2024-11-14 07:24:38,909 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 07:24:38,924 main DEBUG Took 0.012746 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 07:24:38,925 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 07:24:38,925 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 07:24:38,927 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 07:24:38,929 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,941 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 07:24:38,955 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,959 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,961 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,962 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,964 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,965 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,966 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,967 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,969 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,971 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 07:24:38,972 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,972 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 07:24:38,974 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 07:24:38,976 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 07:24:38,979 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 07:24:38,979 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 07:24:38,981 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 07:24:38,982 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 07:24:38,993 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 07:24:38,996 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 07:24:38,998 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 07:24:38,999 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 07:24:38,999 main DEBUG createAppenders(={Console}) 2024-11-14 07:24:39,000 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 07:24:39,001 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 07:24:39,001 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 07:24:39,002 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 07:24:39,002 main DEBUG OutputStream closed 2024-11-14 07:24:39,002 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 07:24:39,003 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 07:24:39,003 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 07:24:39,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 07:24:39,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 07:24:39,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 07:24:39,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 07:24:39,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 07:24:39,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 07:24:39,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 07:24:39,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 07:24:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 07:24:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 07:24:39,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 07:24:39,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 07:24:39,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 07:24:39,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 07:24:39,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 07:24:39,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 07:24:39,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 07:24:39,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 07:24:39,113 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 07:24:39,113 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 07:24:39,114 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 07:24:39,114 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T07:24:39,416 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52 2024-11-14 07:24:39,420 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 07:24:39,420 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T07:24:39,430 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-14T07:24:39,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T07:24:39,471 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d, deleteOnExit=true 2024-11-14T07:24:39,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T07:24:39,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/test.cache.data in system properties and HBase conf 2024-11-14T07:24:39,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T07:24:39,474 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.log.dir in system properties and HBase conf 2024-11-14T07:24:39,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T07:24:39,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T07:24:39,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T07:24:39,587 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T07:24:39,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T07:24:39,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T07:24:39,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T07:24:39,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T07:24:39,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T07:24:39,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T07:24:39,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T07:24:39,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T07:24:39,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T07:24:39,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T07:24:39,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/nfs.dump.dir in system properties and HBase conf 2024-11-14T07:24:39,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/java.io.tmpdir in system properties and HBase conf 2024-11-14T07:24:39,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T07:24:39,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T07:24:39,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T07:24:41,104 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T07:24:41,217 INFO [Time-limited test {}] log.Log(170): Logging initialized @3200ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T07:24:41,332 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:24:41,427 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T07:24:41,464 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T07:24:41,464 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T07:24:41,466 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T07:24:41,482 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:24:41,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.log.dir/,AVAILABLE} 2024-11-14T07:24:41,487 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T07:24:41,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/java.io.tmpdir/jetty-localhost-42875-hadoop-hdfs-3_4_1-tests_jar-_-any-3160648772162707752/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T07:24:41,718 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42875} 2024-11-14T07:24:41,719 INFO [Time-limited test {}] server.Server(415): Started @3703ms 2024-11-14T07:24:42,541 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T07:24:42,551 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T07:24:42,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T07:24:42,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T07:24:42,553 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T07:24:42,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.log.dir/,AVAILABLE} 2024-11-14T07:24:42,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T07:24:42,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/java.io.tmpdir/jetty-localhost-46191-hadoop-hdfs-3_4_1-tests_jar-_-any-1718557279039254760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T07:24:42,679 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:46191} 2024-11-14T07:24:42,679 INFO [Time-limited test {}] server.Server(415): Started @4664ms 2024-11-14T07:24:42,734 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T07:24:43,782 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data2/current/BP-1617860635-172.17.0.2-1731569080462/current, will proceed with Du for space computation calculation, 2024-11-14T07:24:43,782 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data1/current/BP-1617860635-172.17.0.2-1731569080462/current, will proceed with Du for space computation calculation, 2024-11-14T07:24:43,809 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T07:24:43,858 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1da113af0a84213d with lease ID 0x94b0ae9033ba20c: Processing first storage report for DS-31072286-5375-41a3-9252-62cc6a29e68e from datanode DatanodeRegistration(127.0.0.1:42715, datanodeUuid=799d9034-1b08-4be6-9f35-4a763b88be90, infoPort=39941, infoSecurePort=0, ipcPort=38865, storageInfo=lv=-57;cid=testClusterID;nsid=635126699;c=1731569080462) 2024-11-14T07:24:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1da113af0a84213d with lease ID 0x94b0ae9033ba20c: from storage DS-31072286-5375-41a3-9252-62cc6a29e68e node DatanodeRegistration(127.0.0.1:42715, datanodeUuid=799d9034-1b08-4be6-9f35-4a763b88be90, infoPort=39941, infoSecurePort=0, ipcPort=38865, storageInfo=lv=-57;cid=testClusterID;nsid=635126699;c=1731569080462), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T07:24:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1da113af0a84213d with lease ID 0x94b0ae9033ba20c: Processing first storage report for DS-8304315a-20ef-4d4c-ba0a-af05b96ee2f8 from datanode DatanodeRegistration(127.0.0.1:42715, datanodeUuid=799d9034-1b08-4be6-9f35-4a763b88be90, infoPort=39941, infoSecurePort=0, ipcPort=38865, storageInfo=lv=-57;cid=testClusterID;nsid=635126699;c=1731569080462) 2024-11-14T07:24:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1da113af0a84213d with lease ID 0x94b0ae9033ba20c: from storage DS-8304315a-20ef-4d4c-ba0a-af05b96ee2f8 node DatanodeRegistration(127.0.0.1:42715, datanodeUuid=799d9034-1b08-4be6-9f35-4a763b88be90, infoPort=39941, infoSecurePort=0, ipcPort=38865, storageInfo=lv=-57;cid=testClusterID;nsid=635126699;c=1731569080462), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T07:24:43,898 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52 2024-11-14T07:24:43,968 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/zookeeper_0, clientPort=60845, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T07:24:43,979 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60845 2024-11-14T07:24:43,990 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:43,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:44,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741825_1001 (size=7) 2024-11-14T07:24:44,619 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 with version=8 2024-11-14T07:24:44,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/hbase-staging 2024-11-14T07:24:44,705 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T07:24:44,906 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c7aa6e2790b3:0 server-side Connection retries=45 2024-11-14T07:24:44,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:44,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:44,922 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T07:24:44,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:44,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T07:24:45,057 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T07:24:45,110 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T07:24:45,119 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T07:24:45,122 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T07:24:45,145 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 35919 (auto-detected) 2024-11-14T07:24:45,146 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T07:24:45,164 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42927 2024-11-14T07:24:45,185 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42927 connecting to ZooKeeper ensemble=127.0.0.1:60845 2024-11-14T07:24:45,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429270x0, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T07:24:45,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42927-0x101383b61db0000 connected 2024-11-14T07:24:45,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:45,392 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:45,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:24:45,409 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59, hbase.cluster.distributed=false 2024-11-14T07:24:45,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T07:24:45,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-14T07:24:45,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42927 2024-11-14T07:24:45,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42927 2024-11-14T07:24:45,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-14T07:24:45,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-14T07:24:45,556 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c7aa6e2790b3:0 server-side Connection retries=45 2024-11-14T07:24:45,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:45,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:45,558 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T07:24:45,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T07:24:45,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T07:24:45,560 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T07:24:45,563 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T07:24:45,564 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46235 2024-11-14T07:24:45,566 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46235 connecting to ZooKeeper ensemble=127.0.0.1:60845 2024-11-14T07:24:45,567 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:45,570 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:45,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462350x0, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T07:24:45,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462350x0, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:24:45,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46235-0x101383b61db0001 connected 2024-11-14T07:24:45,596 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T07:24:45,604 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T07:24:45,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T07:24:45,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T07:24:45,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46235 2024-11-14T07:24:45,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46235 2024-11-14T07:24:45,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46235 2024-11-14T07:24:45,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46235 2024-11-14T07:24:45,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46235 2024-11-14T07:24:45,632 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c7aa6e2790b3:42927 2024-11-14T07:24:45,633 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:45,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:24:45,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:24:45,650 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:45,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:45,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T07:24:45,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:45,681 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T07:24:45,683 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c7aa6e2790b3,42927,1731569084750 from backup master directory 2024-11-14T07:24:45,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:24:45,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:45,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T07:24:45,698 WARN [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T07:24:45,698 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:45,700 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T07:24:45,701 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T07:24:45,752 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/hbase.id] with ID: b6ed1909-abed-4236-8083-e38d7f3d917f 2024-11-14T07:24:45,752 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/.tmp/hbase.id 2024-11-14T07:24:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741826_1002 (size=42) 2024-11-14T07:24:46,167 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/.tmp/hbase.id]:[hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/hbase.id] 2024-11-14T07:24:46,214 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T07:24:46,221 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T07:24:46,238 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-14T07:24:46,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:46,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:46,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741827_1003 (size=196) 2024-11-14T07:24:46,816 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:24:46,821 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T07:24:46,840 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:46,844 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:24:46,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741828_1004 (size=1189) 2024-11-14T07:24:47,295 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store 2024-11-14T07:24:47,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741829_1005 (size=34) 2024-11-14T07:24:47,720 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T07:24:47,724 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:47,725 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T07:24:47,726 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:24:47,726 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:24:47,728 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T07:24:47,728 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:24:47,728 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:24:47,729 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731569087725Disabling compacts and flushes for region at 1731569087725Disabling writes for close at 1731569087728 (+3 ms)Writing region close event to WAL at 1731569087728Closed at 1731569087728 2024-11-14T07:24:47,732 WARN [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/.initializing 2024-11-14T07:24:47,732 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/WALs/c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:47,742 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T07:24:47,760 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C42927%2C1731569084750, suffix=, logDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/WALs/c7aa6e2790b3,42927,1731569084750, archiveDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/oldWALs, maxLogs=10 2024-11-14T07:24:47,785 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/WALs/c7aa6e2790b3,42927,1731569084750/c7aa6e2790b3%2C42927%2C1731569084750.1731569087765, exclude list is [], retry=0 2024-11-14T07:24:47,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42715,DS-31072286-5375-41a3-9252-62cc6a29e68e,DISK] 2024-11-14T07:24:47,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T07:24:47,850 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/WALs/c7aa6e2790b3,42927,1731569084750/c7aa6e2790b3%2C42927%2C1731569084750.1731569087765 2024-11-14T07:24:47,851 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941)] 2024-11-14T07:24:47,852 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:24:47,853 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:47,858 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,859 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T07:24:47,947 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:47,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:47,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T07:24:47,956 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:47,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:47,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T07:24:47,962 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:47,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:47,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T07:24:47,967 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:47,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:47,969 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,973 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,975 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,982 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,983 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,988 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T07:24:47,994 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T07:24:47,999 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:24:48,001 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60584349, jitterRate=-0.09722284972667694}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T07:24:48,009 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731569087877Initializing all the Stores at 1731569087880 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569087880Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569087881 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569087881Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569087882 (+1 ms)Cleaning up temporary data from old regions at 1731569087983 (+101 ms)Region opened successfully at 1731569088009 (+26 ms) 2024-11-14T07:24:48,011 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T07:24:48,053 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcc19d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c7aa6e2790b3/172.17.0.2:0 2024-11-14T07:24:48,094 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T07:24:48,109 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T07:24:48,109 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T07:24:48,113 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T07:24:48,115 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T07:24:48,121 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T07:24:48,121 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T07:24:48,157 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T07:24:48,168 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T07:24:48,233 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T07:24:48,236 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T07:24:48,237 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T07:24:48,248 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T07:24:48,250 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T07:24:48,254 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T07:24:48,265 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T07:24:48,267 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T07:24:48,279 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T07:24:48,303 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T07:24:48,311 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T07:24:48,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T07:24:48,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T07:24:48,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,325 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c7aa6e2790b3,42927,1731569084750, sessionid=0x101383b61db0000, setting cluster-up flag (Was=false) 2024-11-14T07:24:48,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,385 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T07:24:48,387 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:48,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:48,444 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T07:24:48,446 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:48,455 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T07:24:48,524 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(746): ClusterId : b6ed1909-abed-4236-8083-e38d7f3d917f 2024-11-14T07:24:48,527 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T07:24:48,545 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T07:24:48,545 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T07:24:48,546 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T07:24:48,556 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T07:24:48,557 DEBUG [RS:0;c7aa6e2790b3:46235 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43157a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c7aa6e2790b3/172.17.0.2:0 2024-11-14T07:24:48,558 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T07:24:48,565 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T07:24:48,573 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c7aa6e2790b3,42927,1731569084750 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T07:24:48,577 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c7aa6e2790b3:46235 2024-11-14T07:24:48,581 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T07:24:48,581 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T07:24:48,582 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T07:24:48,582 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:24:48,582 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:24:48,582 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:24:48,582 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T07:24:48,583 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c7aa6e2790b3:0, corePoolSize=10, maxPoolSize=10 2024-11-14T07:24:48,583 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,583 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T07:24:48,583 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,585 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(2659): reportForDuty to master=c7aa6e2790b3,42927,1731569084750 with port=46235, startcode=1731569085523 2024-11-14T07:24:48,593 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731569118592 2024-11-14T07:24:48,595 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T07:24:48,596 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T07:24:48,596 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:24:48,596 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T07:24:48,599 DEBUG [RS:0;c7aa6e2790b3:46235 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T07:24:48,600 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T07:24:48,601 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T07:24:48,601 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T07:24:48,601 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T07:24:48,603 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:48,604 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T07:24:48,612 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,625 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T07:24:48,626 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T07:24:48,626 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T07:24:48,632 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T07:24:48,633 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T07:24:48,636 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731569088635,5,FailOnTimeoutGroup] 2024-11-14T07:24:48,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741831_1007 (size=1321) 2024-11-14T07:24:48,640 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731569088636,5,FailOnTimeoutGroup] 2024-11-14T07:24:48,640 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,641 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T07:24:48,643 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,643 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,644 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T07:24:48,645 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:24:48,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52467, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T07:24:48,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42927 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,673 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42927 {}] master.ServerManager(517): Registering regionserver=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741832_1008 (size=32) 2024-11-14T07:24:48,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:48,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T07:24:48,696 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:24:48,696 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37887 2024-11-14T07:24:48,696 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T07:24:48,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T07:24:48,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:48,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:48,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T07:24:48,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T07:24:48,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:48,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:48,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T07:24:48,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T07:24:48,715 DEBUG [RS:0;c7aa6e2790b3:46235 {}] zookeeper.ZKUtil(111): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,715 WARN [RS:0;c7aa6e2790b3:46235 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T07:24:48,716 INFO [RS:0;c7aa6e2790b3:46235 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:24:48,716 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T07:24:48,718 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:48,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:48,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T07:24:48,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T07:24:48,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:48,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:48,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T07:24:48,730 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c7aa6e2790b3,46235,1731569085523] 2024-11-14T07:24:48,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740 2024-11-14T07:24:48,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740 2024-11-14T07:24:48,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T07:24:48,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T07:24:48,739 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:24:48,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T07:24:48,758 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:24:48,759 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63561609, jitterRate=-0.05285821855068207}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:24:48,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731569088685Initializing all the Stores at 1731569088692 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569088692Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569088693 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569088693Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569088693Cleaning up temporary data from old regions at 1731569088737 (+44 ms)Region opened successfully at 1731569088765 (+28 ms) 2024-11-14T07:24:48,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T07:24:48,765 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T07:24:48,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T07:24:48,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T07:24:48,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T07:24:48,768 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T07:24:48,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731569088765Disabling compacts and flushes for region at 1731569088765Disabling writes for close at 1731569088766 (+1 ms)Writing region close event to WAL at 1731569088768 (+2 ms)Closed at 1731569088768 2024-11-14T07:24:48,773 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:24:48,773 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T07:24:48,777 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T07:24:48,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T07:24:48,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T07:24:48,797 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T07:24:48,804 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T07:24:48,812 INFO [RS:0;c7aa6e2790b3:46235 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T07:24:48,812 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,814 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T07:24:48,819 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T07:24:48,820 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,821 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,821 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,821 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,821 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,821 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c7aa6e2790b3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c7aa6e2790b3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T07:24:48,822 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c7aa6e2790b3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T07:24:48,823 DEBUG [RS:0;c7aa6e2790b3:46235 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T07:24:48,824 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,824 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,824 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,824 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,824 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,825 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,46235,1731569085523-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T07:24:48,853 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T07:24:48,855 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,46235,1731569085523-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,855 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,855 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.Replication(171): c7aa6e2790b3,46235,1731569085523 started 2024-11-14T07:24:48,880 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:48,881 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1482): Serving as c7aa6e2790b3,46235,1731569085523, RpcServer on c7aa6e2790b3/172.17.0.2:46235, sessionid=0x101383b61db0001 2024-11-14T07:24:48,882 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T07:24:48,882 DEBUG [RS:0;c7aa6e2790b3:46235 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,882 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c7aa6e2790b3,46235,1731569085523' 2024-11-14T07:24:48,882 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T07:24:48,883 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T07:24:48,884 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T07:24:48,884 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T07:24:48,885 DEBUG [RS:0;c7aa6e2790b3:46235 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:48,885 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c7aa6e2790b3,46235,1731569085523' 2024-11-14T07:24:48,885 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T07:24:48,886 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T07:24:48,886 DEBUG [RS:0;c7aa6e2790b3:46235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T07:24:48,886 INFO [RS:0;c7aa6e2790b3:46235 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T07:24:48,887 INFO [RS:0;c7aa6e2790b3:46235 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T07:24:48,948 WARN [c7aa6e2790b3:42927 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T07:24:48,991 INFO [RS:0;c7aa6e2790b3:46235 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T07:24:48,995 INFO [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C46235%2C1731569085523, suffix=, logDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523, archiveDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/oldWALs, maxLogs=32 2024-11-14T07:24:49,012 DEBUG [RS:0;c7aa6e2790b3:46235 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523/c7aa6e2790b3%2C46235%2C1731569085523.1731569088998, exclude list is [], retry=0 2024-11-14T07:24:49,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42715,DS-31072286-5375-41a3-9252-62cc6a29e68e,DISK] 2024-11-14T07:24:49,021 INFO [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523/c7aa6e2790b3%2C46235%2C1731569085523.1731569088998 2024-11-14T07:24:49,021 DEBUG [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941)] 2024-11-14T07:24:49,200 DEBUG [c7aa6e2790b3:42927 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T07:24:49,211 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:49,216 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c7aa6e2790b3,46235,1731569085523, state=OPENING 2024-11-14T07:24:49,268 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T07:24:49,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:49,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:24:49,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:24:49,280 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:24:49,282 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T07:24:49,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:24:49,456 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T07:24:49,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33519, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T07:24:49,471 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T07:24:49,471 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T07:24:49,472 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T07:24:49,476 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c7aa6e2790b3%2C46235%2C1731569085523.meta, suffix=.meta, logDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523, archiveDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/oldWALs, maxLogs=32 2024-11-14T07:24:49,496 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523/c7aa6e2790b3%2C46235%2C1731569085523.meta.1731569089478.meta, exclude list is [], retry=0 2024-11-14T07:24:49,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42715,DS-31072286-5375-41a3-9252-62cc6a29e68e,DISK] 2024-11-14T07:24:49,503 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/WALs/c7aa6e2790b3,46235,1731569085523/c7aa6e2790b3%2C46235%2C1731569085523.meta.1731569089478.meta 2024-11-14T07:24:49,504 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941)] 2024-11-14T07:24:49,504 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:24:49,505 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T07:24:49,508 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T07:24:49,513 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T07:24:49,518 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T07:24:49,519 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:49,519 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T07:24:49,519 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T07:24:49,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T07:24:49,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T07:24:49,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:49,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:49,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T07:24:49,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T07:24:49,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:49,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:49,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T07:24:49,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T07:24:49,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:49,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:49,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T07:24:49,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T07:24:49,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:49,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T07:24:49,533 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T07:24:49,535 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740 2024-11-14T07:24:49,538 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740 2024-11-14T07:24:49,540 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T07:24:49,540 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T07:24:49,541 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:24:49,544 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T07:24:49,546 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71641437, jitterRate=0.06754060089588165}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:24:49,546 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T07:24:49,548 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731569089520Writing region info on filesystem at 1731569089520Initializing all the Stores at 1731569089522 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569089522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569089523 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569089523Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731569089523Cleaning up temporary data from old regions at 1731569089540 (+17 ms)Running coprocessor post-open hooks at 1731569089546 (+6 ms)Region opened successfully at 1731569089548 (+2 ms) 2024-11-14T07:24:49,555 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731569089448 2024-11-14T07:24:49,564 DEBUG [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T07:24:49,565 INFO [RS_OPEN_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T07:24:49,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:49,569 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c7aa6e2790b3,46235,1731569085523, state=OPEN 2024-11-14T07:24:49,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T07:24:49,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T07:24:49,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:24:49,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T07:24:49,633 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:49,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T07:24:49,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c7aa6e2790b3,46235,1731569085523 in 350 msec 2024-11-14T07:24:49,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T07:24:49,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 860 msec 2024-11-14T07:24:49,650 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T07:24:49,650 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T07:24:49,670 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:49,671 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:49,689 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:49,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:49,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2120 sec 2024-11-14T07:24:49,711 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731569089711, completionTime=-1 2024-11-14T07:24:49,715 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T07:24:49,715 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T07:24:49,739 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T07:24:49,740 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731569149739 2024-11-14T07:24:49,740 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731569209740 2024-11-14T07:24:49,740 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-14T07:24:49,742 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,742 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,742 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,743 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c7aa6e2790b3:42927, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,744 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,744 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,749 DEBUG [master/c7aa6e2790b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T07:24:49,768 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.070sec 2024-11-14T07:24:49,770 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T07:24:49,771 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T07:24:49,772 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T07:24:49,773 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T07:24:49,773 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T07:24:49,774 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T07:24:49,774 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T07:24:49,782 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T07:24:49,783 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T07:24:49,783 INFO [master/c7aa6e2790b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c7aa6e2790b3,42927,1731569084750-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T07:24:49,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:49,835 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T07:24:49,835 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T07:24:49,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:49,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:49,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db5a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:49,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:49,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:49,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:49,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:49,861 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:49,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:49,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:49,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=320, ProcessCount=11, AvailableMemoryMB=7126 2024-11-14T07:24:49,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:24:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-11-14T07:24:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:24:49,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:24:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:24:49,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:24:49,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:24:49,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T07:24:49,975 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:49,978 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:49,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741835_1011 (size=959) 2024-11-14T07:24:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:50,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:50,400 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:24:50,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741836_1012 (size=53) 2024-11-14T07:24:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:50,810 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:50,810 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eb8aa6a5cb82db71eb9aa5187f031ed4, disabling compactions & flushes 2024-11-14T07:24:50,810 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:50,810 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:50,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. after waiting 0 ms 2024-11-14T07:24:50,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:50,811 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:50,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb8aa6a5cb82db71eb9aa5187f031ed4: Waiting for close lock at 1731569090810Disabling compacts and flushes for region at 1731569090810Disabling writes for close at 1731569090811 (+1 ms)Writing region close event to WAL at 1731569090811Closed at 1731569090811 2024-11-14T07:24:50,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:24:50,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569090814"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569090814"}]},"ts":"1731569090814"} 2024-11-14T07:24:50,824 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:24:50,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:24:50,830 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569090827"}]},"ts":"1731569090827"} 2024-11-14T07:24:50,835 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:24:50,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, ASSIGN}] 2024-11-14T07:24:50,840 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, ASSIGN 2024-11-14T07:24:50,843 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:24:50,994 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb8aa6a5cb82db71eb9aa5187f031ed4, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:50,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, ASSIGN because future has completed 2024-11-14T07:24:51,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:24:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:51,161 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:51,161 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:24:51,162 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,162 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:24:51,162 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,162 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,164 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,166 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb8aa6a5cb82db71eb9aa5187f031ed4 columnFamilyName A 2024-11-14T07:24:51,167 DEBUG [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:51,168 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(327): Store=eb8aa6a5cb82db71eb9aa5187f031ed4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:51,168 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,170 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb8aa6a5cb82db71eb9aa5187f031ed4 columnFamilyName B 2024-11-14T07:24:51,171 DEBUG [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:51,172 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(327): Store=eb8aa6a5cb82db71eb9aa5187f031ed4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:51,172 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,174 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb8aa6a5cb82db71eb9aa5187f031ed4 columnFamilyName C 2024-11-14T07:24:51,175 DEBUG [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:24:51,176 INFO [StoreOpener-eb8aa6a5cb82db71eb9aa5187f031ed4-1 {}] regionserver.HStore(327): Store=eb8aa6a5cb82db71eb9aa5187f031ed4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:24:51,176 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,177 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,178 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,181 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,181 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,182 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:24:51,185 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,189 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:24:51,190 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eb8aa6a5cb82db71eb9aa5187f031ed4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74511673, jitterRate=0.11031045019626617}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:24:51,191 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:51,192 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eb8aa6a5cb82db71eb9aa5187f031ed4: Running coprocessor pre-open hook at 1731569091162Writing region info on filesystem at 1731569091162Initializing all the Stores at 1731569091164 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569091164Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569091164Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569091164Cleaning up temporary data from old regions at 1731569091181 (+17 ms)Running coprocessor post-open hooks at 1731569091191 (+10 ms)Region opened successfully at 1731569091192 (+1 ms) 2024-11-14T07:24:51,194 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., pid=6, masterSystemTime=1731569091154 2024-11-14T07:24:51,199 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:51,199 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:51,200 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb8aa6a5cb82db71eb9aa5187f031ed4, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:51,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:24:51,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T07:24:51,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 in 207 msec 2024-11-14T07:24:51,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T07:24:51,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, ASSIGN in 376 msec 2024-11-14T07:24:51,221 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:24:51,222 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569091222"}]},"ts":"1731569091222"} 2024-11-14T07:24:51,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:24:51,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:24:51,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2680 sec 2024-11-14T07:24:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T07:24:52,139 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:24:52,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,153 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1cdbfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,156 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@972c14f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,167 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:24:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:24:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1432456c 2024-11-14T07:24:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:24:52,173 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:24:52,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f89bd3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fba9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,215 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,220 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,227 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7489f107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333dd90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c24f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:24:52,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:24:52,239 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:24:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:24:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:24:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2341aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:24:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:24:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:24:52,242 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:24:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fb5a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:24:52,247 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,247 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,247 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,249 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,249 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,250 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,254 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:24:52,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,254 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,261 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,270 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,270 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:24:52,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T07:24:52,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:24:52,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:24:52,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:24:52,277 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:24:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:24:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:52,282 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:24:52,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:24:52,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:52,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:24:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:52,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d91cfb834d4c4933bdd07a41460c86b1 is 50, key is test_row_0/A:col10/1731569092346/Put/seqid=0 2024-11-14T07:24:52,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:52,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:52,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741837_1013 (size=12001) 2024-11-14T07:24:52,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d91cfb834d4c4933bdd07a41460c86b1 2024-11-14T07:24:52,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569152554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569152562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569152562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569152567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569152562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a is 50, key is test_row_0/B:col10/1731569092346/Put/seqid=0 2024-11-14T07:24:52,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:52,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741838_1014 (size=12001) 2024-11-14T07:24:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569152718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569152718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569152719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569152718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569152720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569152927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569152936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569152937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569152938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569152938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:52,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:52,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a 2024-11-14T07:24:53,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/35f32ac8857f41588ea1ae6f0e538b2b is 50, key is test_row_0/C:col10/1731569092346/Put/seqid=0 2024-11-14T07:24:53,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741839_1015 (size=12001) 2024-11-14T07:24:53,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/35f32ac8857f41588ea1ae6f0e538b2b 2024-11-14T07:24:53,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d91cfb834d4c4933bdd07a41460c86b1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1 2024-11-14T07:24:53,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:24:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a 2024-11-14T07:24:53,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:24:53,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/35f32ac8857f41588ea1ae6f0e538b2b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b 2024-11-14T07:24:53,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T07:24:53,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 880ms, sequenceid=13, compaction requested=false 2024-11-14T07:24:53,238 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T07:24:53,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:53,246 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:53,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:24:53,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:53,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/853f4a68f34f4d7ca9455f4df52945e0 is 50, key is test_row_0/A:col10/1731569093241/Put/seqid=0 2024-11-14T07:24:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569153298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569153302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741840_1016 (size=14341) 2024-11-14T07:24:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569153304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/853f4a68f34f4d7ca9455f4df52945e0 2024-11-14T07:24:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569153306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569153309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/ac365b78798a4db5b7885a8d81eaa548 is 50, key is test_row_0/B:col10/1731569093241/Put/seqid=0 2024-11-14T07:24:53,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741841_1017 (size=12001) 2024-11-14T07:24:53,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/ac365b78798a4db5b7885a8d81eaa548 2024-11-14T07:24:53,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/dc190a87634f410f8ae79af18b24b209 is 50, key is test_row_0/C:col10/1731569093241/Put/seqid=0 2024-11-14T07:24:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:53,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569153427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569153429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569153436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569153445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569153455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741842_1018 (size=12001) 2024-11-14T07:24:53,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/dc190a87634f410f8ae79af18b24b209 2024-11-14T07:24:53,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/853f4a68f34f4d7ca9455f4df52945e0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0 2024-11-14T07:24:53,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0, entries=200, sequenceid=38, filesize=14.0 K 2024-11-14T07:24:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/ac365b78798a4db5b7885a8d81eaa548 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548 2024-11-14T07:24:53,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:24:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/dc190a87634f410f8ae79af18b24b209 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209 2024-11-14T07:24:53,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:24:53,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 314ms, sequenceid=38, compaction requested=false 2024-11-14T07:24:53,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:53,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T07:24:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:53,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:24:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/71e3115fd00f46579c76036a4251eb27 is 50, key is test_row_0/A:col10/1731569093304/Put/seqid=0 2024-11-14T07:24:53,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:53,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741843_1019 (size=12001) 2024-11-14T07:24:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569153771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569153777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569153781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569153800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569153806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569153905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569153921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569153922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569153921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:53,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569153922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:53,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:53,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/71e3115fd00f46579c76036a4251eb27 2024-11-14T07:24:54,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e0cb76febf334e719d76ef0e78bdc7b6 is 50, key is test_row_0/B:col10/1731569093304/Put/seqid=0 2024-11-14T07:24:54,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741844_1020 (size=12001) 2024-11-14T07:24:54,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e0cb76febf334e719d76ef0e78bdc7b6 2024-11-14T07:24:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569154136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/656d156722734553ab68e9c381b1b48e is 50, key is test_row_0/C:col10/1731569093304/Put/seqid=0 2024-11-14T07:24:54,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569154145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569154155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569154155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569154156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741845_1021 (size=12001) 2024-11-14T07:24:54,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/656d156722734553ab68e9c381b1b48e 2024-11-14T07:24:54,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/71e3115fd00f46579c76036a4251eb27 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27 2024-11-14T07:24:54,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T07:24:54,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e0cb76febf334e719d76ef0e78bdc7b6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6 2024-11-14T07:24:54,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T07:24:54,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/656d156722734553ab68e9c381b1b48e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e 2024-11-14T07:24:54,289 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T07:24:54,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 706ms, sequenceid=49, compaction requested=true 2024-11-14T07:24:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T07:24:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T07:24:54,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T07:24:54,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-11-14T07:24:54,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.0490 sec 2024-11-14T07:24:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T07:24:54,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:24:54,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:24:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T07:24:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:54,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:24:54,431 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:24:54,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:24:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:54,452 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:54,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T07:24:54,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b056e19e1a27404285dc7f3bb6cf4fa0 is 50, key is test_row_0/A:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569154475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569154479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569154486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569154487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569154479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741846_1022 (size=14341) 2024-11-14T07:24:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:54,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:54,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569154595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569154605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569154606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569154606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:54,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569154805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569154827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569154828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569154828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:54,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:54,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b056e19e1a27404285dc7f3bb6cf4fa0 2024-11-14T07:24:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:54,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:54,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e2307ae1817d4c6986867f82f63f2172 is 50, key is test_row_0/B:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:54,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741847_1023 (size=12001) 2024-11-14T07:24:54,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e2307ae1817d4c6986867f82f63f2172 2024-11-14T07:24:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569154995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1778951427d74dcba83d99d77232f5a4 is 50, key is test_row_0/C:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:55,033 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:24:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:55,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741848_1024 (size=12001) 2024-11-14T07:24:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1778951427d74dcba83d99d77232f5a4 2024-11-14T07:24:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b056e19e1a27404285dc7f3bb6cf4fa0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0 2024-11-14T07:24:55,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0, entries=200, sequenceid=76, filesize=14.0 K 2024-11-14T07:24:55,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e2307ae1817d4c6986867f82f63f2172 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172 2024-11-14T07:24:55,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569155116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T07:24:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1778951427d74dcba83d99d77232f5a4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4 2024-11-14T07:24:55,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T07:24:55,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 693ms, sequenceid=76, compaction requested=true 2024-11-14T07:24:55,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:55,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:24:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:24:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:24:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:55,162 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:55,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:55,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:24:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:55,165 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:55,171 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:55,173 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:24:55,174 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:55,174 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:24:55,174 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,175 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=46.9 K 2024-11-14T07:24:55,175 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,176 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=51.4 K 2024-11-14T07:24:55,177 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e6dc5e1d1cb44227a5a4bbc3d6ce4d8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731569092323 2024-11-14T07:24:55,178 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91cfb834d4c4933bdd07a41460c86b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731569092323 2024-11-14T07:24:55,179 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ac365b78798a4db5b7885a8d81eaa548, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569092553 2024-11-14T07:24:55,180 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853f4a68f34f4d7ca9455f4df52945e0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569092553 2024-11-14T07:24:55,181 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e0cb76febf334e719d76ef0e78bdc7b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731569093287 2024-11-14T07:24:55,181 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e3115fd00f46579c76036a4251eb27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731569093287 2024-11-14T07:24:55,182 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T07:24:55,183 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b056e19e1a27404285dc7f3bb6cf4fa0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093766 2024-11-14T07:24:55,183 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e2307ae1817d4c6986867f82f63f2172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093775 2024-11-14T07:24:55,184 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T07:24:55,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/fa42b88c42644e30b491b46594b10404 is 50, key is test_row_0/A:col10/1731569094481/Put/seqid=0 2024-11-14T07:24:55,184 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T07:24:55,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T07:24:55,189 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T07:24:55,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T07:24:55,189 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T07:24:55,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T07:24:55,191 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T07:24:55,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741849_1025 (size=14341) 2024-11-14T07:24:55,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/fa42b88c42644e30b491b46594b10404 2024-11-14T07:24:55,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e03bf39193e446539406411ac44c5d78 is 50, key is test_row_0/B:col10/1731569094481/Put/seqid=0 2024-11-14T07:24:55,273 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#13 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:55,274 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f498efc2fad54f23932310faf49527fa is 50, key is test_row_0/A:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:55,283 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#14 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:55,285 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/5c8d278566164e3590e4f6462889f5da is 50, key is test_row_0/B:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:55,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741850_1026 (size=12001) 2024-11-14T07:24:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741851_1027 (size=12139) 2024-11-14T07:24:55,360 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f498efc2fad54f23932310faf49527fa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f498efc2fad54f23932310faf49527fa 2024-11-14T07:24:55,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741852_1028 (size=12139) 2024-11-14T07:24:55,376 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/5c8d278566164e3590e4f6462889f5da as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/5c8d278566164e3590e4f6462889f5da 2024-11-14T07:24:55,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569155338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569155344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569155341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,402 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into f498efc2fad54f23932310faf49527fa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:55,402 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 5c8d278566164e3590e4f6462889f5da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:55,402 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:55,402 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:55,405 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569095147; duration=0sec 2024-11-14T07:24:55,406 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:55,407 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569095160; duration=0sec 2024-11-14T07:24:55,407 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:24:55,407 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:55,407 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:24:55,407 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:55,412 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:55,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:24:55,413 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,413 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=46.9 K 2024-11-14T07:24:55,415 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f32ac8857f41588ea1ae6f0e538b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731569092323 2024-11-14T07:24:55,416 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc190a87634f410f8ae79af18b24b209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569092553 2024-11-14T07:24:55,418 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656d156722734553ab68e9c381b1b48e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731569093287 2024-11-14T07:24:55,419 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1778951427d74dcba83d99d77232f5a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093775 2024-11-14T07:24:55,484 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#16 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:55,487 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/8fd68a53fe414765a10f733337127d47 is 50, key is test_row_0/C:col10/1731569093781/Put/seqid=0 2024-11-14T07:24:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569155484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569155485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569155485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741853_1029 (size=12139) 2024-11-14T07:24:55,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569155628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:55,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569155704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569155705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569155705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:55,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e03bf39193e446539406411ac44c5d78 2024-11-14T07:24:55,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/18e20658c7e94cf8b9846a244159381d is 50, key is test_row_0/C:col10/1731569094481/Put/seqid=0 2024-11-14T07:24:55,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741854_1030 (size=12001) 2024-11-14T07:24:55,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/18e20658c7e94cf8b9846a244159381d 2024-11-14T07:24:55,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/fa42b88c42644e30b491b46594b10404 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404 2024-11-14T07:24:55,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404, entries=200, sequenceid=88, filesize=14.0 K 2024-11-14T07:24:55,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/e03bf39193e446539406411ac44c5d78 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78 2024-11-14T07:24:55,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T07:24:55,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/18e20658c7e94cf8b9846a244159381d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d 2024-11-14T07:24:55,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T07:24:55,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 659ms, sequenceid=88, compaction requested=false 2024-11-14T07:24:55,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:55,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T07:24:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:55,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:24:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a8ced28e44064049a61e3c8325dbadd4 is 50, key is test_row_0/A:col10/1731569095337/Put/seqid=0 2024-11-14T07:24:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741855_1031 (size=12001) 2024-11-14T07:24:55,876 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a8ced28e44064049a61e3c8325dbadd4 2024-11-14T07:24:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/61176a38bc90450abb375b6d8c76a4d8 is 50, key is test_row_0/B:col10/1731569095337/Put/seqid=0 2024-11-14T07:24:55,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741856_1032 (size=12001) 2024-11-14T07:24:55,907 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/61176a38bc90450abb375b6d8c76a4d8 2024-11-14T07:24:55,924 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/8fd68a53fe414765a10f733337127d47 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8fd68a53fe414765a10f733337127d47 2024-11-14T07:24:55,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/d386ab692fe840ef87434aa2d3f49a8b is 50, key is test_row_0/C:col10/1731569095337/Put/seqid=0 2024-11-14T07:24:55,946 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 8fd68a53fe414765a10f733337127d47(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:55,947 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:55,947 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569095160; duration=0sec 2024-11-14T07:24:55,947 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:55,947 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:24:55,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741857_1033 (size=12001) 2024-11-14T07:24:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/d386ab692fe840ef87434aa2d3f49a8b 2024-11-14T07:24:55,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a8ced28e44064049a61e3c8325dbadd4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4 2024-11-14T07:24:55,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T07:24:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/61176a38bc90450abb375b6d8c76a4d8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8 2024-11-14T07:24:56,009 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T07:24:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/d386ab692fe840ef87434aa2d3f49a8b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b 2024-11-14T07:24:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:56,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:56,025 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T07:24:56,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 179ms, sequenceid=114, compaction requested=true 2024-11-14T07:24:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T07:24:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T07:24:56,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T07:24:56,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6000 sec 2024-11-14T07:24:56,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6100 sec 2024-11-14T07:24:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:56,043 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:56,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:24:56,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ad99620ff1044c47a1aacc77ec7c8a12 is 50, key is test_row_0/A:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741858_1034 (size=19021) 2024-11-14T07:24:56,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ad99620ff1044c47a1aacc77ec7c8a12 2024-11-14T07:24:56,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/f167303a57b54deb92d984d83a47b65b is 50, key is test_row_0/B:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569156120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569156130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741859_1035 (size=12001) 2024-11-14T07:24:56,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/f167303a57b54deb92d984d83a47b65b 2024-11-14T07:24:56,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/e96f3936b89e445e911cd7a1322aa3dd is 50, key is test_row_0/C:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741860_1036 (size=12001) 2024-11-14T07:24:56,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/e96f3936b89e445e911cd7a1322aa3dd 2024-11-14T07:24:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ad99620ff1044c47a1aacc77ec7c8a12 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12 2024-11-14T07:24:56,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12, entries=300, sequenceid=126, filesize=18.6 K 2024-11-14T07:24:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/f167303a57b54deb92d984d83a47b65b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b 2024-11-14T07:24:56,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T07:24:56,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/e96f3936b89e445e911cd7a1322aa3dd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd 2024-11-14T07:24:56,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569156235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd, entries=150, sequenceid=126, filesize=11.7 K 2024-11-14T07:24:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569156245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 207ms, sequenceid=126, compaction requested=true 2024-11-14T07:24:56,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:24:56,251 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:56,251 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:24:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:56,253 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:56,254 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:24:56,254 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,254 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/5c8d278566164e3590e4f6462889f5da, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=47.0 K 2024-11-14T07:24:56,255 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:56,255 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:24:56,255 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,255 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f498efc2fad54f23932310faf49527fa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=56.2 K 2024-11-14T07:24:56,256 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c8d278566164e3590e4f6462889f5da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093775 2024-11-14T07:24:56,256 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e03bf39193e446539406411ac44c5d78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569094474 2024-11-14T07:24:56,257 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f498efc2fad54f23932310faf49527fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093775 2024-11-14T07:24:56,258 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 61176a38bc90450abb375b6d8c76a4d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731569095291 2024-11-14T07:24:56,259 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa42b88c42644e30b491b46594b10404, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569094474 2024-11-14T07:24:56,259 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f167303a57b54deb92d984d83a47b65b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096034 2024-11-14T07:24:56,260 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ced28e44064049a61e3c8325dbadd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731569095291 2024-11-14T07:24:56,262 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad99620ff1044c47a1aacc77ec7c8a12, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096018 2024-11-14T07:24:56,292 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:56,293 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/91ef286135b34726b6c8d170e3cbc4fb is 50, key is test_row_0/B:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,304 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:56,305 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/90c35a71ee954c44b3a67424fc695dcc is 50, key is test_row_0/A:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741861_1037 (size=12275) 2024-11-14T07:24:56,329 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/91ef286135b34726b6c8d170e3cbc4fb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/91ef286135b34726b6c8d170e3cbc4fb 2024-11-14T07:24:56,341 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 91ef286135b34726b6c8d170e3cbc4fb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:56,341 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,341 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569096251; duration=0sec 2024-11-14T07:24:56,342 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:56,342 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:24:56,342 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:24:56,346 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:24:56,346 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:24:56,347 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,347 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8fd68a53fe414765a10f733337127d47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=47.0 K 2024-11-14T07:24:56,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741862_1038 (size=12275) 2024-11-14T07:24:56,349 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd68a53fe414765a10f733337127d47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731569093775 2024-11-14T07:24:56,352 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 18e20658c7e94cf8b9846a244159381d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569094474 2024-11-14T07:24:56,353 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d386ab692fe840ef87434aa2d3f49a8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731569095291 2024-11-14T07:24:56,354 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e96f3936b89e445e911cd7a1322aa3dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096034 2024-11-14T07:24:56,359 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/90c35a71ee954c44b3a67424fc695dcc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/90c35a71ee954c44b3a67424fc695dcc 2024-11-14T07:24:56,372 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 90c35a71ee954c44b3a67424fc695dcc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:56,372 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,372 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569096251; duration=0sec 2024-11-14T07:24:56,373 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:56,373 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:24:56,379 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:56,380 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/d445624c6ec749b5a17fb2237d00b28f is 50, key is test_row_0/C:col10/1731569096034/Put/seqid=0 2024-11-14T07:24:56,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741863_1039 (size=12275) 2024-11-14T07:24:56,418 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/d445624c6ec749b5a17fb2237d00b28f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d445624c6ec749b5a17fb2237d00b28f 2024-11-14T07:24:56,434 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into d445624c6ec749b5a17fb2237d00b28f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:56,434 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,434 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569096251; duration=0sec 2024-11-14T07:24:56,435 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:56,436 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:24:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:56,450 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:56,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:24:56,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d1dfb8f1b9d04d8295b6c8c648124d09 is 50, key is test_row_0/A:col10/1731569096446/Put/seqid=0 2024-11-14T07:24:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569156470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569156474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741864_1040 (size=12151) 2024-11-14T07:24:56,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d1dfb8f1b9d04d8295b6c8c648124d09 2024-11-14T07:24:56,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/34b33a022a0e4642a7d3647805700f59 is 50, key is test_row_0/B:col10/1731569096446/Put/seqid=0 2024-11-14T07:24:56,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741865_1041 (size=12151) 2024-11-14T07:24:56,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/34b33a022a0e4642a7d3647805700f59 2024-11-14T07:24:56,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/da1e039009244ecfbe0d579b24cb7900 is 50, key is test_row_0/C:col10/1731569096446/Put/seqid=0 2024-11-14T07:24:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741866_1042 (size=12151) 2024-11-14T07:24:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T07:24:56,585 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:24:56,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:24:56,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/da1e039009244ecfbe0d579b24cb7900 2024-11-14T07:24:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T07:24:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569156586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569156587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,597 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:24:56,599 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:24:56,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:24:56,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/d1dfb8f1b9d04d8295b6c8c648124d09 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09 2024-11-14T07:24:56,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T07:24:56,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/34b33a022a0e4642a7d3647805700f59 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59 2024-11-14T07:24:56,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T07:24:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/da1e039009244ecfbe0d579b24cb7900 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900 2024-11-14T07:24:56,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T07:24:56,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 190ms, sequenceid=156, compaction requested=false 2024-11-14T07:24:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:56,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:56,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:24:56,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6f6932416eff47f4ac5604b80b284d03 is 50, key is test_row_0/A:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:56,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741867_1043 (size=12151) 2024-11-14T07:24:56,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6f6932416eff47f4ac5604b80b284d03 2024-11-14T07:24:56,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:24:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/b8df62cf85334eefa9c9bf85544c27b6 is 50, key is test_row_0/B:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:56,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741868_1044 (size=12151) 2024-11-14T07:24:56,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/b8df62cf85334eefa9c9bf85544c27b6 2024-11-14T07:24:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569156806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569156806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569156807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/313d4a5066ea47f1a173d483870c83bc is 50, key is test_row_0/C:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:56,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741869_1045 (size=12151) 2024-11-14T07:24:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:56,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:24:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:56,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569156924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569156924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569156924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:56,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:24:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569157125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569157125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569157135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569157135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569157135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:57,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:24:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:57,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/313d4a5066ea47f1a173d483870c83bc 2024-11-14T07:24:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6f6932416eff47f4ac5604b80b284d03 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03 2024-11-14T07:24:57,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T07:24:57,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/b8df62cf85334eefa9c9bf85544c27b6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6 2024-11-14T07:24:57,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T07:24:57,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/313d4a5066ea47f1a173d483870c83bc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc 2024-11-14T07:24:57,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T07:24:57,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 688ms, sequenceid=167, compaction requested=true 2024-11-14T07:24:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:24:57,350 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:24:57,350 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:24:57,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:57,352 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:57,352 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:24:57,352 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,352 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/91ef286135b34726b6c8d170e3cbc4fb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=35.7 K 2024-11-14T07:24:57,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:57,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:24:57,353 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,353 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/90c35a71ee954c44b3a67424fc695dcc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=35.7 K 2024-11-14T07:24:57,354 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ef286135b34726b6c8d170e3cbc4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096034 2024-11-14T07:24:57,355 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c35a71ee954c44b3a67424fc695dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096034 2024-11-14T07:24:57,355 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b33a022a0e4642a7d3647805700f59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731569096116 2024-11-14T07:24:57,355 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1dfb8f1b9d04d8295b6c8c648124d09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731569096116 2024-11-14T07:24:57,356 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b8df62cf85334eefa9c9bf85544c27b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:57,356 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6932416eff47f4ac5604b80b284d03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:57,379 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:57,379 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:57,380 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b40f46422af04fc8a31db15c0718b977 is 50, key is test_row_0/A:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:57,380 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1b0710f2587c42a1b0c1ee4cd6760490 is 50, key is test_row_0/B:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:57,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T07:24:57,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,382 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:24:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7237b7b85af44745940f4aef3f613282 is 50, key is test_row_0/A:col10/1731569096811/Put/seqid=0 2024-11-14T07:24:57,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741870_1046 (size=12527) 2024-11-14T07:24:57,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741871_1047 (size=12527) 2024-11-14T07:24:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741872_1048 (size=12151) 2024-11-14T07:24:57,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7237b7b85af44745940f4aef3f613282 2024-11-14T07:24:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:57,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bb0b253748da45f08c98f46642d98fe6 is 50, key is test_row_0/B:col10/1731569096811/Put/seqid=0 2024-11-14T07:24:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569157472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569157481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569157482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741873_1049 (size=12151) 2024-11-14T07:24:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569157595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569157636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569157645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569157805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569157815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569157815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:57,826 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b40f46422af04fc8a31db15c0718b977 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b40f46422af04fc8a31db15c0718b977 2024-11-14T07:24:57,837 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1b0710f2587c42a1b0c1ee4cd6760490 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1b0710f2587c42a1b0c1ee4cd6760490 2024-11-14T07:24:57,844 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into b40f46422af04fc8a31db15c0718b977(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:57,844 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:57,845 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569097350; duration=0sec 2024-11-14T07:24:57,846 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:57,847 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:24:57,847 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:57,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:57,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:24:57,849 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:57,849 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d445624c6ec749b5a17fb2237d00b28f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=35.7 K 2024-11-14T07:24:57,850 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 1b0710f2587c42a1b0c1ee4cd6760490(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:57,850 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:57,850 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569097350; duration=0sec 2024-11-14T07:24:57,850 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d445624c6ec749b5a17fb2237d00b28f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731569096034 2024-11-14T07:24:57,850 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:57,851 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:24:57,851 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1e039009244ecfbe0d579b24cb7900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731569096116 2024-11-14T07:24:57,853 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313d4a5066ea47f1a173d483870c83bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:57,868 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:57,869 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/701aa74c50b2442f803d0eb287006acb is 50, key is test_row_0/C:col10/1731569096469/Put/seqid=0 2024-11-14T07:24:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741874_1050 (size=12527) 2024-11-14T07:24:57,915 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bb0b253748da45f08c98f46642d98fe6 2024-11-14T07:24:57,929 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/701aa74c50b2442f803d0eb287006acb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/701aa74c50b2442f803d0eb287006acb 2024-11-14T07:24:57,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fda15365a4dd44f4990c93ef27cccdc6 is 50, key is test_row_0/C:col10/1731569096811/Put/seqid=0 2024-11-14T07:24:57,944 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 701aa74c50b2442f803d0eb287006acb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:57,944 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:57,944 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=13, startTime=1731569097351; duration=0sec 2024-11-14T07:24:57,944 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:57,944 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:24:57,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741875_1051 (size=12151) 2024-11-14T07:24:57,951 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fda15365a4dd44f4990c93ef27cccdc6 2024-11-14T07:24:57,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7237b7b85af44745940f4aef3f613282 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282 2024-11-14T07:24:57,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T07:24:57,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bb0b253748da45f08c98f46642d98fe6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6 2024-11-14T07:24:57,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T07:24:57,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fda15365a4dd44f4990c93ef27cccdc6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6 2024-11-14T07:24:58,006 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T07:24:58,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 625ms, sequenceid=192, compaction requested=false 2024-11-14T07:24:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T07:24:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T07:24:58,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T07:24:58,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-14T07:24:58,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4320 sec 2024-11-14T07:24:58,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:58,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:24:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:58,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ecfe50913304953a91bf1f15919614a is 50, key is test_row_0/A:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741876_1052 (size=12151) 2024-11-14T07:24:58,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ecfe50913304953a91bf1f15919614a 2024-11-14T07:24:58,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/64d554dc62664558908162f4716db5a6 is 50, key is test_row_0/B:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741877_1053 (size=12151) 2024-11-14T07:24:58,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569158216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569158227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569158223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569158335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569158336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569158344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569158545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569158554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569158565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/64d554dc62664558908162f4716db5a6 2024-11-14T07:24:58,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/9db390f72a59497f98386cb54dcba34c is 50, key is test_row_0/C:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569158654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,659 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in eb8aa6a5cb82db71eb9aa5187f031ed4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:24:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569158665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,667 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in eb8aa6a5cb82db71eb9aa5187f031ed4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:24:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741878_1054 (size=12151) 2024-11-14T07:24:58,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/9db390f72a59497f98386cb54dcba34c 2024-11-14T07:24:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ecfe50913304953a91bf1f15919614a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a 2024-11-14T07:24:58,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T07:24:58,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/64d554dc62664558908162f4716db5a6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6 2024-11-14T07:24:58,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T07:24:58,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/9db390f72a59497f98386cb54dcba34c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c 2024-11-14T07:24:58,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T07:24:58,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 604ms, sequenceid=208, compaction requested=true 2024-11-14T07:24:58,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:58,732 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:58,733 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:24:58,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:58,734 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:58,734 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:24:58,734 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T07:24:58,734 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b40f46422af04fc8a31db15c0718b977, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.0 K 2024-11-14T07:24:58,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:24:58,736 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:58,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:24:58,736 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40f46422af04fc8a31db15c0718b977, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:58,736 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:24:58,737 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,737 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1b0710f2587c42a1b0c1ee4cd6760490, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.0 K 2024-11-14T07:24:58,737 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7237b7b85af44745940f4aef3f613282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731569096792 2024-11-14T07:24:58,738 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b0710f2587c42a1b0c1ee4cd6760490, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T07:24:58,738 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ecfe50913304953a91bf1f15919614a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:24:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:24:58,741 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:24:58,741 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0b253748da45f08c98f46642d98fe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731569096792 2024-11-14T07:24:58,742 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d554dc62664558908162f4716db5a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:24:58,745 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:24:58,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:24:58,783 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:58,783 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/53cc7b8dca9b40c4b982737b1f3d5d69 is 50, key is test_row_0/A:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,790 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:58,791 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/4f3c87d9e99d459e9522db738935ea2a is 50, key is test_row_0/B:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741880_1056 (size=12629) 2024-11-14T07:24:58,844 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/4f3c87d9e99d459e9522db738935ea2a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4f3c87d9e99d459e9522db738935ea2a 2024-11-14T07:24:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:24:58,854 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 4f3c87d9e99d459e9522db738935ea2a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:58,854 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:58,854 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569098733; duration=0sec 2024-11-14T07:24:58,855 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:24:58,855 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:24:58,855 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:24:58,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741879_1055 (size=12629) 2024-11-14T07:24:58,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:24:58,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:24:58,858 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,859 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/701aa74c50b2442f803d0eb287006acb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.0 K 2024-11-14T07:24:58,859 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 701aa74c50b2442f803d0eb287006acb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569096468 2024-11-14T07:24:58,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fda15365a4dd44f4990c93ef27cccdc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731569096792 2024-11-14T07:24:58,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db390f72a59497f98386cb54dcba34c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:24:58,866 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/53cc7b8dca9b40c4b982737b1f3d5d69 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/53cc7b8dca9b40c4b982737b1f3d5d69 2024-11-14T07:24:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:24:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:24:58,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:24:58,882 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 53cc7b8dca9b40c4b982737b1f3d5d69(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:58,883 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:58,883 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569098732; duration=0sec 2024-11-14T07:24:58,883 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:58,883 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:24:58,887 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:24:58,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/197f049c799a4b70894bddf57eb5bf36 is 50, key is test_row_0/A:col10/1731569098170/Put/seqid=0 2024-11-14T07:24:58,887 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/3675a89b60ee453ebecaab34007dfc6f is 50, key is test_row_0/C:col10/1731569097472/Put/seqid=0 2024-11-14T07:24:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569158898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:58,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569158901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569158902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:58,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741881_1057 (size=12629) 2024-11-14T07:24:58,952 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/3675a89b60ee453ebecaab34007dfc6f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/3675a89b60ee453ebecaab34007dfc6f 2024-11-14T07:24:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741882_1058 (size=14541) 2024-11-14T07:24:58,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/197f049c799a4b70894bddf57eb5bf36 2024-11-14T07:24:58,964 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 3675a89b60ee453ebecaab34007dfc6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:24:58,964 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:58,964 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=13, startTime=1731569098733; duration=0sec 2024-11-14T07:24:58,965 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:24:58,965 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:24:58,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/4a26ef9d15414dc999c45d5749f0e7a0 is 50, key is test_row_0/B:col10/1731569098170/Put/seqid=0 2024-11-14T07:24:59,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569159005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741883_1059 (size=12151) 2024-11-14T07:24:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569159015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569159015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:24:59,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569159215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569159225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569159235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:24:59,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:59,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/4a26ef9d15414dc999c45d5749f0e7a0 2024-11-14T07:24:59,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/f1cc543399574868a41d2993d2925c44 is 50, key is test_row_0/C:col10/1731569098170/Put/seqid=0 2024-11-14T07:24:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741884_1060 (size=12151) 2024-11-14T07:24:59,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/f1cc543399574868a41d2993d2925c44 2024-11-14T07:24:59,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/197f049c799a4b70894bddf57eb5bf36 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36 2024-11-14T07:24:59,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36, entries=200, sequenceid=234, filesize=14.2 K 2024-11-14T07:24:59,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/4a26ef9d15414dc999c45d5749f0e7a0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0 2024-11-14T07:24:59,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:24:59,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:24:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569159525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:24:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569159535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/f1cc543399574868a41d2993d2925c44 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44 2024-11-14T07:24:59,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:24:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569159544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:24:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:24:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:24:59,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:24:59,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 684ms, sequenceid=234, compaction requested=false 2024-11-14T07:24:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:59,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T07:24:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:24:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/8eedaeeb2d154d4db1c590d9184b804b is 50, key is test_row_0/A:col10/1731569098900/Put/seqid=0 2024-11-14T07:24:59,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741885_1061 (size=9757) 2024-11-14T07:24:59,705 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/8eedaeeb2d154d4db1c590d9184b804b 2024-11-14T07:24:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/c07248a9691e4fe8875a2048fe8875c1 is 50, key is test_row_0/B:col10/1731569098900/Put/seqid=0 2024-11-14T07:24:59,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741886_1062 (size=9757) 2024-11-14T07:24:59,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/c07248a9691e4fe8875a2048fe8875c1 2024-11-14T07:24:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a1ae5bfd3a2449798cc21f891b06b85b is 50, key is test_row_0/C:col10/1731569098900/Put/seqid=0 2024-11-14T07:24:59,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741887_1063 (size=9757) 2024-11-14T07:24:59,802 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a1ae5bfd3a2449798cc21f891b06b85b 2024-11-14T07:24:59,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/8eedaeeb2d154d4db1c590d9184b804b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b 2024-11-14T07:24:59,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b, entries=100, sequenceid=248, filesize=9.5 K 2024-11-14T07:24:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/c07248a9691e4fe8875a2048fe8875c1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1 2024-11-14T07:24:59,866 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1, entries=100, sequenceid=248, filesize=9.5 K 2024-11-14T07:24:59,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a1ae5bfd3a2449798cc21f891b06b85b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b 2024-11-14T07:24:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:24:59,878 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b, entries=100, sequenceid=248, filesize=9.5 K 2024-11-14T07:24:59,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 205ms, sequenceid=248, compaction requested=true 2024-11-14T07:24:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:24:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:24:59,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T07:24:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T07:24:59,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T07:24:59,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1380 sec 2024-11-14T07:24:59,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1510 sec 2024-11-14T07:25:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:00,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:00,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:00,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/cd995180ed4b4e01a59246fcad2f49bd is 50, key is test_row_0/A:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741888_1064 (size=14641) 2024-11-14T07:25:00,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/cd995180ed4b4e01a59246fcad2f49bd 2024-11-14T07:25:00,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8b67354a7ee64de7bd19d37b43194791 is 50, key is test_row_0/B:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569160111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569160117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569160119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741889_1065 (size=12201) 2024-11-14T07:25:00,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8b67354a7ee64de7bd19d37b43194791 2024-11-14T07:25:00,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f7f1c7dd7a74e75a3c35291ac137be0 is 50, key is test_row_0/C:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741890_1066 (size=12201) 2024-11-14T07:25:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569160225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569160226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569160234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569160435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569160436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569160445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f7f1c7dd7a74e75a3c35291ac137be0 2024-11-14T07:25:00,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/cd995180ed4b4e01a59246fcad2f49bd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd 2024-11-14T07:25:00,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd, entries=200, sequenceid=259, filesize=14.3 K 2024-11-14T07:25:00,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8b67354a7ee64de7bd19d37b43194791 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791 2024-11-14T07:25:00,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T07:25:00,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f7f1c7dd7a74e75a3c35291ac137be0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0 2024-11-14T07:25:00,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T07:25:00,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 544ms, sequenceid=259, compaction requested=true 2024-11-14T07:25:00,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:00,600 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:00,600 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:00,602 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46738 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:00,602 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:00,603 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:00,603 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:00,603 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:00,603 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:00,603 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/53cc7b8dca9b40c4b982737b1f3d5d69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=50.4 K 2024-11-14T07:25:00,603 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4f3c87d9e99d459e9522db738935ea2a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=45.6 K 2024-11-14T07:25:00,603 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3c87d9e99d459e9522db738935ea2a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:25:00,604 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cc7b8dca9b40c4b982737b1f3d5d69, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:25:00,604 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a26ef9d15414dc999c45d5749f0e7a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569098170 2024-11-14T07:25:00,604 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 197f049c799a4b70894bddf57eb5bf36, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569098170 2024-11-14T07:25:00,607 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eedaeeb2d154d4db1c590d9184b804b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731569098896 2024-11-14T07:25:00,607 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c07248a9691e4fe8875a2048fe8875c1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731569098896 2024-11-14T07:25:00,607 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b67354a7ee64de7bd19d37b43194791, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:00,607 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd995180ed4b4e01a59246fcad2f49bd, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:00,625 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:00,626 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/b483ea60fe214e24841ac5deb34ab001 is 50, key is test_row_0/B:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,627 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:00,627 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/82a6bf89d85a4453bed158733d984985 is 50, key is test_row_0/A:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741892_1068 (size=12815) 2024-11-14T07:25:00,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741891_1067 (size=12815) 2024-11-14T07:25:00,663 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/b483ea60fe214e24841ac5deb34ab001 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b483ea60fe214e24841ac5deb34ab001 2024-11-14T07:25:00,673 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into b483ea60fe214e24841ac5deb34ab001(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:00,673 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:00,673 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569100600; duration=0sec 2024-11-14T07:25:00,674 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:00,674 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:00,674 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:00,676 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46738 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:00,676 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:00,677 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:00,679 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/3675a89b60ee453ebecaab34007dfc6f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=45.6 K 2024-11-14T07:25:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:00,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:00,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:25:00,682 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3675a89b60ee453ebecaab34007dfc6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731569097472 2024-11-14T07:25:00,685 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f1cc543399574868a41d2993d2925c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569098170 2024-11-14T07:25:00,685 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ae5bfd3a2449798cc21f891b06b85b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731569098896 2024-11-14T07:25:00,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/0c2cdde8f872422f9027450f7dc86eda is 50, key is test_row_0/A:col10/1731569100115/Put/seqid=0 2024-11-14T07:25:00,689 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f7f1c7dd7a74e75a3c35291ac137be0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:00,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569160705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741893_1069 (size=14741) 2024-11-14T07:25:00,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569160707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/0c2cdde8f872422f9027450f7dc86eda 2024-11-14T07:25:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,714 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:00,715 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/def8ee88074c469cb6cdc73121ce1dff is 50, key is test_row_0/C:col10/1731569100054/Put/seqid=0 2024-11-14T07:25:00,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8be62445acf24666adff2519ab54a394 is 50, key is test_row_0/B:col10/1731569100115/Put/seqid=0 2024-11-14T07:25:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741894_1070 (size=12815) 2024-11-14T07:25:00,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741895_1071 (size=12301) 2024-11-14T07:25:00,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8be62445acf24666adff2519ab54a394 2024-11-14T07:25:00,741 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/def8ee88074c469cb6cdc73121ce1dff as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/def8ee88074c469cb6cdc73121ce1dff 2024-11-14T07:25:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569160745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569160745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,753 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into def8ee88074c469cb6cdc73121ce1dff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:00,753 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:00,753 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569100600; duration=0sec 2024-11-14T07:25:00,754 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:00,754 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569160755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7c8f1e807c7c43d69f316715e27b8645 is 50, key is test_row_0/C:col10/1731569100115/Put/seqid=0 2024-11-14T07:25:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741896_1072 (size=12301) 2024-11-14T07:25:00,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569160815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569160817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T07:25:00,885 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:00,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T07:25:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:00,890 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:00,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:00,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569161025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569161025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,047 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/82a6bf89d85a4453bed158733d984985 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/82a6bf89d85a4453bed158733d984985 2024-11-14T07:25:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,056 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 82a6bf89d85a4453bed158733d984985(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:01,056 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:01,056 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569100599; duration=0sec 2024-11-14T07:25:01,057 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:01,057 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:01,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7c8f1e807c7c43d69f316715e27b8645 2024-11-14T07:25:01,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/0c2cdde8f872422f9027450f7dc86eda as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda 2024-11-14T07:25:01,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda, entries=200, sequenceid=286, filesize=14.4 K 2024-11-14T07:25:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/8be62445acf24666adff2519ab54a394 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394 2024-11-14T07:25:01,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394, entries=150, sequenceid=286, filesize=12.0 K 2024-11-14T07:25:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7c8f1e807c7c43d69f316715e27b8645 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645 2024-11-14T07:25:01,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645, entries=150, sequenceid=286, filesize=12.0 K 2024-11-14T07:25:01,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 544ms, sequenceid=286, compaction requested=false 2024-11-14T07:25:01,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:01,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:01,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:01,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c1a6344e47d34e40a723dea9854f63e3 is 50, key is test_row_0/A:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741897_1073 (size=12301) 2024-11-14T07:25:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569161313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569161314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569161315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569161335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569161335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569161424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569161424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569161424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569161634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569161634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569161635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c1a6344e47d34e40a723dea9854f63e3 2024-11-14T07:25:01,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 is 50, key is test_row_0/B:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:01,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741898_1074 (size=12301) 2024-11-14T07:25:01,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569161844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569161844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569161944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569161944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569161945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:01,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:01,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:02,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 2024-11-14T07:25:02,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/6eb8c9b93b6e4a6c9abae35e699cb623 is 50, key is test_row_0/C:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:02,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741899_1075 (size=12301) 2024-11-14T07:25:02,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/6eb8c9b93b6e4a6c9abae35e699cb623 2024-11-14T07:25:02,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c1a6344e47d34e40a723dea9854f63e3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3 2024-11-14T07:25:02,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:02,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:02,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T07:25:02,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 2024-11-14T07:25:02,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T07:25:02,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/6eb8c9b93b6e4a6c9abae35e699cb623 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623 2024-11-14T07:25:02,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T07:25:02,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 893ms, sequenceid=299, compaction requested=true 2024-11-14T07:25:02,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:02,151 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:02,151 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:02,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:02,152 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:02,152 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:02,152 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,152 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/82a6bf89d85a4453bed158733d984985, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=38.9 K 2024-11-14T07:25:02,152 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:02,152 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:02,153 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,153 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b483ea60fe214e24841ac5deb34ab001, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.5 K 2024-11-14T07:25:02,153 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b483ea60fe214e24841ac5deb34ab001, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:02,153 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a6bf89d85a4453bed158733d984985, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:02,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2cdde8f872422f9027450f7dc86eda, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731569100106 2024-11-14T07:25:02,154 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be62445acf24666adff2519ab54a394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731569100106 2024-11-14T07:25:02,154 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd46a1eb2e847f4a3eac5c06f01b4d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:02,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a6344e47d34e40a723dea9854f63e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:02,164 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:02,164 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:02,164 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/72bdd8a5d74f4ad38a688ec75697907f is 50, key is test_row_0/A:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:02,165 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/abeb284ee53b421d9256ca38b2b6ff64 is 50, key is test_row_0/B:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:02,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741900_1076 (size=13017) 2024-11-14T07:25:02,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741901_1077 (size=13017) 2024-11-14T07:25:02,182 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/abeb284ee53b421d9256ca38b2b6ff64 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/abeb284ee53b421d9256ca38b2b6ff64 2024-11-14T07:25:02,190 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into abeb284ee53b421d9256ca38b2b6ff64(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:02,191 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:02,191 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569102151; duration=0sec 2024-11-14T07:25:02,191 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:02,191 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:02,191 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:02,193 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:02,193 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:02,193 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,193 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/def8ee88074c469cb6cdc73121ce1dff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.5 K 2024-11-14T07:25:02,194 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting def8ee88074c469cb6cdc73121ce1dff, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731569100043 2024-11-14T07:25:02,194 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8f1e807c7c43d69f316715e27b8645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731569100106 2024-11-14T07:25:02,194 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb8c9b93b6e4a6c9abae35e699cb623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:02,209 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:02,210 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/4ac95cfa900c49c0b481e7ab480b4b6c is 50, key is test_row_0/C:col10/1731569100701/Put/seqid=0 2024-11-14T07:25:02,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741902_1078 (size=13017) 2024-11-14T07:25:02,234 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/4ac95cfa900c49c0b481e7ab480b4b6c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4ac95cfa900c49c0b481e7ab480b4b6c 2024-11-14T07:25:02,242 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 4ac95cfa900c49c0b481e7ab480b4b6c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:02,242 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:02,243 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=13, startTime=1731569102151; duration=0sec 2024-11-14T07:25:02,243 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:02,243 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:02,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T07:25:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,286 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:25:02,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/68ddccede5324f369922bab97084075c is 50, key is test_row_0/A:col10/1731569101311/Put/seqid=0 2024-11-14T07:25:02,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741903_1079 (size=12301) 2024-11-14T07:25:02,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/68ddccede5324f369922bab97084075c 2024-11-14T07:25:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/042f935afff94336a256b2eb90c15af8 is 50, key is test_row_0/B:col10/1731569101311/Put/seqid=0 2024-11-14T07:25:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741904_1080 (size=12301) 2024-11-14T07:25:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/042f935afff94336a256b2eb90c15af8 2024-11-14T07:25:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5521d3b7aad445d2b46d55a83ae78006 is 50, key is test_row_0/C:col10/1731569101311/Put/seqid=0 2024-11-14T07:25:02,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741905_1081 (size=12301) 2024-11-14T07:25:02,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5521d3b7aad445d2b46d55a83ae78006 2024-11-14T07:25:02,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/68ddccede5324f369922bab97084075c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c 2024-11-14T07:25:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T07:25:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/042f935afff94336a256b2eb90c15af8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8 2024-11-14T07:25:02,381 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T07:25:02,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5521d3b7aad445d2b46d55a83ae78006 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006 2024-11-14T07:25:02,390 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006, entries=150, sequenceid=327, filesize=12.0 K 2024-11-14T07:25:02,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 106ms, sequenceid=327, compaction requested=false 2024-11-14T07:25:02,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:02,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:02,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T07:25:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T07:25:02,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T07:25:02,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-14T07:25:02,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5300 sec 2024-11-14T07:25:02,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:02,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:02,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b46100911f754d938624d88cf954075f is 50, key is test_row_0/A:col10/1731569102468/Put/seqid=0 2024-11-14T07:25:02,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741906_1082 (size=12301) 2024-11-14T07:25:02,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b46100911f754d938624d88cf954075f 2024-11-14T07:25:02,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/14438bcf6e3a41018c737b9c6d43fad0 is 50, key is test_row_0/B:col10/1731569102468/Put/seqid=0 2024-11-14T07:25:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569162521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569162521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569162524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741907_1083 (size=12301) 2024-11-14T07:25:02,586 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/72bdd8a5d74f4ad38a688ec75697907f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/72bdd8a5d74f4ad38a688ec75697907f 2024-11-14T07:25:02,592 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 72bdd8a5d74f4ad38a688ec75697907f(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:02,592 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:02,592 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569102150; duration=0sec 2024-11-14T07:25:02,592 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:02,592 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569162635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569162635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569162635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569162845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569162845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569162845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569162855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,856 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in eb8aa6a5cb82db71eb9aa5187f031ed4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569162864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:02,868 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in eb8aa6a5cb82db71eb9aa5187f031ed4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:02,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/14438bcf6e3a41018c737b9c6d43fad0 2024-11-14T07:25:02,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fc723fe6ef374e9097b48039a34b6674 is 50, key is test_row_0/C:col10/1731569102468/Put/seqid=0 2024-11-14T07:25:02,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741908_1084 (size=12301) 2024-11-14T07:25:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T07:25:03,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:03,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T07:25:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:25:03,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:03,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:03,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569163154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569163154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569163155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:25:03,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T07:25:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:03,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fc723fe6ef374e9097b48039a34b6674 2024-11-14T07:25:03,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b46100911f754d938624d88cf954075f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f 2024-11-14T07:25:03,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T07:25:03,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/14438bcf6e3a41018c737b9c6d43fad0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0 2024-11-14T07:25:03,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T07:25:03,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/fc723fe6ef374e9097b48039a34b6674 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674 2024-11-14T07:25:03,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T07:25:03,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:03,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:25:03,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T07:25:03,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 918ms, sequenceid=339, compaction requested=true 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:03,386 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:03,386 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:03,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:03,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:03,388 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,388 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,388 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/abeb284ee53b421d9256ca38b2b6ff64, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.7 K 2024-11-14T07:25:03,388 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/72bdd8a5d74f4ad38a688ec75697907f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=36.7 K 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bdd8a5d74f4ad38a688ec75697907f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:03,388 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting abeb284ee53b421d9256ca38b2b6ff64, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:03,389 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 042f935afff94336a256b2eb90c15af8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731569101289 2024-11-14T07:25:03,389 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ddccede5324f369922bab97084075c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731569101289 2024-11-14T07:25:03,389 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 14438bcf6e3a41018c737b9c6d43fad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731569102462 2024-11-14T07:25:03,389 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46100911f754d938624d88cf954075f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731569102462 2024-11-14T07:25:03,398 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:03,398 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:03,399 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/05af92a78a4e4eb0b7fd0b6822d91147 is 50, key is test_row_0/A:col10/1731569102468/Put/seqid=0 2024-11-14T07:25:03,399 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1e476785c642424182a68631f23e6bf3 is 50, key is test_row_0/B:col10/1731569102468/Put/seqid=0 2024-11-14T07:25:03,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741909_1085 (size=13119) 2024-11-14T07:25:03,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741910_1086 (size=13119) 2024-11-14T07:25:03,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T07:25:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:25:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ebcc8f978d8f46c194b10a25f782f023 is 50, key is test_row_0/A:col10/1731569102522/Put/seqid=0 2024-11-14T07:25:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741911_1087 (size=12301) 2024-11-14T07:25:03,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ebcc8f978d8f46c194b10a25f782f023 2024-11-14T07:25:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bbd8d470b14d47b2b046f2db22cdafc2 is 50, key is test_row_0/B:col10/1731569102522/Put/seqid=0 2024-11-14T07:25:03,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741912_1088 (size=12301) 2024-11-14T07:25:03,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bbd8d470b14d47b2b046f2db22cdafc2 2024-11-14T07:25:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/20c1939326144493b221039c5edde33c is 50, key is test_row_0/C:col10/1731569102522/Put/seqid=0 2024-11-14T07:25:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741913_1089 (size=12301) 2024-11-14T07:25:03,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/20c1939326144493b221039c5edde33c 2024-11-14T07:25:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/ebcc8f978d8f46c194b10a25f782f023 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023 2024-11-14T07:25:03,594 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T07:25:03,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/bbd8d470b14d47b2b046f2db22cdafc2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2 2024-11-14T07:25:03,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T07:25:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/20c1939326144493b221039c5edde33c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c 2024-11-14T07:25:03,610 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T07:25:03,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 82ms, sequenceid=364, compaction requested=true 2024-11-14T07:25:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T07:25:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T07:25:03,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T07:25:03,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 543 msec 2024-11-14T07:25:03,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 581 msec 2024-11-14T07:25:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:03,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:03,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:03,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/4f87661c09de41a4a1382d96e63685f8 is 50, key is test_row_1/A:col10/1731569103671/Put/seqid=0 2024-11-14T07:25:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741914_1090 (size=14737) 2024-11-14T07:25:03,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/4f87661c09de41a4a1382d96e63685f8 2024-11-14T07:25:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T07:25:03,695 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:03,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T07:25:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:25:03,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/fe299a363aaf4c65aefba109e1aa452b is 50, key is test_row_1/B:col10/1731569103671/Put/seqid=0 2024-11-14T07:25:03,699 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:03,700 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:03,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:03,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569163715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569163717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569163718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741915_1091 (size=9857) 2024-11-14T07:25:03,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/fe299a363aaf4c65aefba109e1aa452b 2024-11-14T07:25:03,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5c206d5350f1446b9675f4b70a9400a0 is 50, key is test_row_1/C:col10/1731569103671/Put/seqid=0 2024-11-14T07:25:03,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741916_1092 (size=9857) 2024-11-14T07:25:03,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5c206d5350f1446b9675f4b70a9400a0 2024-11-14T07:25:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/4f87661c09de41a4a1382d96e63685f8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8 2024-11-14T07:25:03,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8, entries=200, sequenceid=375, filesize=14.4 K 2024-11-14T07:25:03,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/fe299a363aaf4c65aefba109e1aa452b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b 2024-11-14T07:25:03,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b, entries=100, sequenceid=375, filesize=9.6 K 2024-11-14T07:25:03,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5c206d5350f1446b9675f4b70a9400a0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0 2024-11-14T07:25:03,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0, entries=100, sequenceid=375, filesize=9.6 K 2024-11-14T07:25:03,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 102ms, sequenceid=375, compaction requested=true 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T07:25:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:25:03,812 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1e476785c642424182a68631f23e6bf3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1e476785c642424182a68631f23e6bf3 2024-11-14T07:25:03,821 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 1e476785c642424182a68631f23e6bf3(size=12.8 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:03,821 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:03,822 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569103386; duration=0sec 2024-11-14T07:25:03,822 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-14T07:25:03,822 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:03,822 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-14T07:25:03,823 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:25:03,825 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:25:03,825 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. because compaction request was cancelled 2024-11-14T07:25:03,825 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:03,826 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-14T07:25:03,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:03,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:25:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:03,828 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/05af92a78a4e4eb0b7fd0b6822d91147 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/05af92a78a4e4eb0b7fd0b6822d91147 2024-11-14T07:25:03,831 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59777 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-14T07:25:03,832 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:03,832 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,833 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4ac95cfa900c49c0b481e7ab480b4b6c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=58.4 K 2024-11-14T07:25:03,833 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac95cfa900c49c0b481e7ab480b4b6c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731569100690 2024-11-14T07:25:03,834 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5521d3b7aad445d2b46d55a83ae78006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731569101289 2024-11-14T07:25:03,834 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fc723fe6ef374e9097b48039a34b6674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731569102462 2024-11-14T07:25:03,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c1939326144493b221039c5edde33c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731569102510 2024-11-14T07:25:03,836 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c206d5350f1446b9675f4b70a9400a0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731569103671 2024-11-14T07:25:03,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 is 50, key is test_row_0/A:col10/1731569103714/Put/seqid=0 2024-11-14T07:25:03,844 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 05af92a78a4e4eb0b7fd0b6822d91147(size=12.8 K), total size for store is 39.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:03,844 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:03,845 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569103386; duration=0sec 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. because compaction request was cancelled 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:03,845 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:03,846 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:03,846 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:03,847 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,847 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1e476785c642424182a68631f23e6bf3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=34.5 K 2024-11-14T07:25:03,847 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e476785c642424182a68631f23e6bf3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731569102462 2024-11-14T07:25:03,848 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd8d470b14d47b2b046f2db22cdafc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731569102510 2024-11-14T07:25:03,850 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe299a363aaf4c65aefba109e1aa452b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731569103671 2024-11-14T07:25:03,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:25:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:03,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:03,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569163852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569163853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569163858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,869 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:03,869 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/6adf1f50b7d642829a863b70be306cab is 50, key is test_row_0/C:col10/1731569102522/Put/seqid=0 2024-11-14T07:25:03,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741917_1093 (size=14741) 2024-11-14T07:25:03,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 2024-11-14T07:25:03,883 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#82 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:03,884 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1c3445ebf3a24949a26fdfcfbc47f82e is 50, key is test_row_0/B:col10/1731569102522/Put/seqid=0 2024-11-14T07:25:03,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/534297c24138478fa9c4247b50f139fe is 50, key is test_row_0/B:col10/1731569103714/Put/seqid=0 2024-11-14T07:25:03,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741919_1095 (size=13221) 2024-11-14T07:25:03,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741918_1094 (size=13187) 2024-11-14T07:25:03,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741920_1096 (size=12301) 2024-11-14T07:25:03,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/534297c24138478fa9c4247b50f139fe 2024-11-14T07:25:03,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1efa0afc280843959aa1765192517f0e is 50, key is test_row_0/C:col10/1731569103714/Put/seqid=0 2024-11-14T07:25:03,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741921_1097 (size=12301) 2024-11-14T07:25:03,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1efa0afc280843959aa1765192517f0e 2024-11-14T07:25:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569163965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569163965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569163965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:03,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:03,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 2024-11-14T07:25:03,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2, entries=200, sequenceid=402, filesize=14.4 K 2024-11-14T07:25:03,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/534297c24138478fa9c4247b50f139fe as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe 2024-11-14T07:25:03,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T07:25:04,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/1efa0afc280843959aa1765192517f0e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e 2024-11-14T07:25:04,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:25:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:25:04,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T07:25:04,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 196ms, sequenceid=402, compaction requested=true 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:04,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T07:25:04,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T07:25:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,162 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a2074183a6fb4d0bac24ee4b55cbb3c1 is 50, key is test_row_0/A:col10/1731569103852/Put/seqid=0 2024-11-14T07:25:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741922_1098 (size=12301) 2024-11-14T07:25:04,187 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a2074183a6fb4d0bac24ee4b55cbb3c1 2024-11-14T07:25:04,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/25dc9f9e5bfe41a68c8d6e65004568e9 is 50, key is test_row_0/B:col10/1731569103852/Put/seqid=0 2024-11-14T07:25:04,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741923_1099 (size=12301) 2024-11-14T07:25:04,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569164224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569164227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569164230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:25:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569164335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569164335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,338 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1c3445ebf3a24949a26fdfcfbc47f82e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1c3445ebf3a24949a26fdfcfbc47f82e 2024-11-14T07:25:04,339 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/6adf1f50b7d642829a863b70be306cab as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6adf1f50b7d642829a863b70be306cab 2024-11-14T07:25:04,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,347 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 1c3445ebf3a24949a26fdfcfbc47f82e(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:04,347 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569164346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,347 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569103774; duration=0sec 2024-11-14T07:25:04,347 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T07:25:04,347 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:04,347 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 5 compacting, 1 eligible, 16 blocking 2024-11-14T07:25:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,348 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:25:04,348 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. because compaction request was cancelled 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:04,349 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 6adf1f50b7d642829a863b70be306cab(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:04,349 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=11, startTime=1731569103774; duration=0sec 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:04,349 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:04,350 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-14T07:25:04,351 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. because compaction request was cancelled 2024-11-14T07:25:04,351 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/05af92a78a4e4eb0b7fd0b6822d91147, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=53.6 K 2024-11-14T07:25:04,351 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:04,352 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05af92a78a4e4eb0b7fd0b6822d91147, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731569102462 2024-11-14T07:25:04,352 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebcc8f978d8f46c194b10a25f782f023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731569102510 2024-11-14T07:25:04,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f87661c09de41a4a1382d96e63685f8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731569103668 2024-11-14T07:25:04,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e8afa6fb5f4bf7a04ef1440072e1a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731569103713 2024-11-14T07:25:04,364 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#87 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:04,365 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/29ceb3c9498346989f1d78816c7b07db is 50, key is test_row_0/A:col10/1731569103714/Put/seqid=0 2024-11-14T07:25:04,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741924_1100 (size=13255) 2024-11-14T07:25:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569164544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569164545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569164555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,613 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/25dc9f9e5bfe41a68c8d6e65004568e9 2024-11-14T07:25:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a76db206f30f481d9a3031ca08614c2f is 50, key is test_row_0/C:col10/1731569103852/Put/seqid=0 2024-11-14T07:25:04,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741925_1101 (size=12301) 2024-11-14T07:25:04,646 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a76db206f30f481d9a3031ca08614c2f 2024-11-14T07:25:04,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/a2074183a6fb4d0bac24ee4b55cbb3c1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1 2024-11-14T07:25:04,672 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T07:25:04,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/25dc9f9e5bfe41a68c8d6e65004568e9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9 2024-11-14T07:25:04,682 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T07:25:04,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a76db206f30f481d9a3031ca08614c2f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f 2024-11-14T07:25:04,692 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T07:25:04,693 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 531ms, sequenceid=414, compaction requested=true 2024-11-14T07:25:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T07:25:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T07:25:04,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T07:25:04,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-14T07:25:04,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0050 sec 2024-11-14T07:25:04,782 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/29ceb3c9498346989f1d78816c7b07db as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/29ceb3c9498346989f1d78816c7b07db 2024-11-14T07:25:04,791 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 29ceb3c9498346989f1d78816c7b07db(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:04,791 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:04,791 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569104024; duration=0sec 2024-11-14T07:25:04,791 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:04,791 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T07:25:04,835 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:04,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T07:25:04,838 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:04,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:04,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:04,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:04,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:25:04,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c42c70af168548258ff89d1534a0f9d0 is 50, key is test_row_0/A:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569164891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569164892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569164892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569164896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569164896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:04,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741926_1102 (size=17181) 2024-11-14T07:25:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:04,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:04,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:04,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:04,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569165004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569165004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569165005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569165005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569165005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:05,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:05,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569165215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569165215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569165215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569165215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569165215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c42c70af168548258ff89d1534a0f9d0 2024-11-14T07:25:05,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/62ab7917cc29436682f8d9de8137ec6b is 50, key is test_row_0/B:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741927_1103 (size=12301) 2024-11-14T07:25:05,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569165525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569165525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569165525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569165525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569165525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:05,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:05,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:05,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/62ab7917cc29436682f8d9de8137ec6b 2024-11-14T07:25:05,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:05,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/526a4f9a3c0d43cfbf41246b230ff7cd is 50, key is test_row_0/C:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:05,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741928_1104 (size=12301) 2024-11-14T07:25:05,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/526a4f9a3c0d43cfbf41246b230ff7cd 2024-11-14T07:25:05,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/c42c70af168548258ff89d1534a0f9d0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0 2024-11-14T07:25:05,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0, entries=250, sequenceid=444, filesize=16.8 K 2024-11-14T07:25:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/62ab7917cc29436682f8d9de8137ec6b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b 2024-11-14T07:25:05,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T07:25:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/526a4f9a3c0d43cfbf41246b230ff7cd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd 2024-11-14T07:25:05,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd, entries=150, sequenceid=444, filesize=12.0 K 2024-11-14T07:25:05,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 995ms, sequenceid=444, compaction requested=true 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:05,856 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:05,856 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:05,858 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:05,858 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:05,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:05,859 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:05,859 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,859 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1c3445ebf3a24949a26fdfcfbc47f82e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=48.9 K 2024-11-14T07:25:05,859 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,859 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/29ceb3c9498346989f1d78816c7b07db, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=41.7 K 2024-11-14T07:25:05,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3445ebf3a24949a26fdfcfbc47f82e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731569102522 2024-11-14T07:25:05,860 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ceb3c9498346989f1d78816c7b07db, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731569103713 2024-11-14T07:25:05,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 534297c24138478fa9c4247b50f139fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731569103713 2024-11-14T07:25:05,861 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2074183a6fb4d0bac24ee4b55cbb3c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731569103849 2024-11-14T07:25:05,861 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 25dc9f9e5bfe41a68c8d6e65004568e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731569103849 2024-11-14T07:25:05,862 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42c70af168548258ff89d1534a0f9d0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:05,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ab7917cc29436682f8d9de8137ec6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:05,877 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:05,878 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f254ee9c45bc4867afd92e0767081e7c is 50, key is test_row_0/A:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:05,882 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:05,882 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/c6cfea64daa74fd3b2bb32fb38c7befe is 50, key is test_row_0/B:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:05,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741929_1105 (size=13357) 2024-11-14T07:25:05,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T07:25:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T07:25:05,924 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f254ee9c45bc4867afd92e0767081e7c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f254ee9c45bc4867afd92e0767081e7c 2024-11-14T07:25:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/5a820422e4994e94bb565eae2ff6759f is 50, key is test_row_0/A:col10/1731569104890/Put/seqid=0 2024-11-14T07:25:05,938 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into f254ee9c45bc4867afd92e0767081e7c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:05,938 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:05,938 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569105856; duration=0sec 2024-11-14T07:25:05,938 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:05,938 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:05,938 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:05,941 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:05,941 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:05,942 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:05,942 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6adf1f50b7d642829a863b70be306cab, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=48.9 K 2024-11-14T07:25:05,942 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6adf1f50b7d642829a863b70be306cab, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731569102522 2024-11-14T07:25:05,942 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1efa0afc280843959aa1765192517f0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731569103713 2024-11-14T07:25:05,943 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76db206f30f481d9a3031ca08614c2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731569103849 2024-11-14T07:25:05,943 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 526a4f9a3c0d43cfbf41246b230ff7cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:05,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741930_1106 (size=13357) 2024-11-14T07:25:05,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741931_1107 (size=12301) 2024-11-14T07:25:05,964 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/5a820422e4994e94bb565eae2ff6759f 2024-11-14T07:25:05,971 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:05,972 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/84d82f1036cb4df584ca6977f4d3ce14 is 50, key is test_row_0/C:col10/1731569104227/Put/seqid=0 2024-11-14T07:25:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/afd0f451687745159a23eb78918fe8b1 is 50, key is test_row_0/B:col10/1731569104890/Put/seqid=0 2024-11-14T07:25:06,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741932_1108 (size=13323) 2024-11-14T07:25:06,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741933_1109 (size=12301) 2024-11-14T07:25:06,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/afd0f451687745159a23eb78918fe8b1 2024-11-14T07:25:06,026 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/84d82f1036cb4df584ca6977f4d3ce14 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/84d82f1036cb4df584ca6977f4d3ce14 2024-11-14T07:25:06,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:06,046 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 84d82f1036cb4df584ca6977f4d3ce14(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:06,046 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,046 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569105856; duration=0sec 2024-11-14T07:25:06,046 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:06,046 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:06,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/4da038e6e292439d8829a59cb3af64fd is 50, key is test_row_0/C:col10/1731569104890/Put/seqid=0 2024-11-14T07:25:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741934_1110 (size=12301) 2024-11-14T07:25:06,115 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/4da038e6e292439d8829a59cb3af64fd 2024-11-14T07:25:06,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/5a820422e4994e94bb565eae2ff6759f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f 2024-11-14T07:25:06,137 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:25:06,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/afd0f451687745159a23eb78918fe8b1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1 2024-11-14T07:25:06,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:25:06,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/4da038e6e292439d8829a59cb3af64fd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd 2024-11-14T07:25:06,165 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd, entries=150, sequenceid=453, filesize=12.0 K 2024-11-14T07:25:06,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 245ms, sequenceid=453, compaction requested=false 2024-11-14T07:25:06,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:06,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T07:25:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T07:25:06,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T07:25:06,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-14T07:25:06,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3360 sec 2024-11-14T07:25:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:06,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:06,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T07:25:06,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/24eda2f8f5ba430895120ec6bd1ef164 is 50, key is test_row_0/A:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741935_1111 (size=12301) 2024-11-14T07:25:06,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/24eda2f8f5ba430895120ec6bd1ef164 2024-11-14T07:25:06,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/a0ab3b8a6eb14f95b788eaa8b811f91e is 50, key is test_row_0/B:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741936_1112 (size=12301) 2024-11-14T07:25:06,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/a0ab3b8a6eb14f95b788eaa8b811f91e 2024-11-14T07:25:06,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a560164161f24101857379e2dc2035c0 is 50, key is test_row_0/C:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741937_1113 (size=12301) 2024-11-14T07:25:06,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a560164161f24101857379e2dc2035c0 2024-11-14T07:25:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/24eda2f8f5ba430895120ec6bd1ef164 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164 2024-11-14T07:25:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164, entries=150, sequenceid=486, filesize=12.0 K 2024-11-14T07:25:06,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/a0ab3b8a6eb14f95b788eaa8b811f91e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e 2024-11-14T07:25:06,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e, entries=150, sequenceid=486, filesize=12.0 K 2024-11-14T07:25:06,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/a560164161f24101857379e2dc2035c0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0 2024-11-14T07:25:06,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0, entries=150, sequenceid=486, filesize=12.0 K 2024-11-14T07:25:06,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 155ms, sequenceid=486, compaction requested=true 2024-11-14T07:25:06,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:06,354 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/c6cfea64daa74fd3b2bb32fb38c7befe as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c6cfea64daa74fd3b2bb32fb38c7befe 2024-11-14T07:25:06,355 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:06,355 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:06,355 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:06,355 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f254ee9c45bc4867afd92e0767081e7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=37.1 K 2024-11-14T07:25:06,356 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f254ee9c45bc4867afd92e0767081e7c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:06,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:06,356 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a820422e4994e94bb565eae2ff6759f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731569104863 2024-11-14T07:25:06,357 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24eda2f8f5ba430895120ec6bd1ef164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:06,365 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into c6cfea64daa74fd3b2bb32fb38c7befe(size=13.0 K), total size for store is 37.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:06,365 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,365 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569105856; duration=0sec 2024-11-14T07:25:06,366 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:06,366 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:06,366 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:06,366 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:06,367 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:06,367 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:06,367 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:06,367 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/84d82f1036cb4df584ca6977f4d3ce14, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=37.0 K 2024-11-14T07:25:06,369 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d82f1036cb4df584ca6977f4d3ce14, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:06,369 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da038e6e292439d8829a59cb3af64fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731569104863 2024-11-14T07:25:06,370 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a560164161f24101857379e2dc2035c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:06,371 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:06,372 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/43d8b8753bd54cc3b442bdc6adfd0470 is 50, key is test_row_0/A:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741938_1114 (size=13459) 2024-11-14T07:25:06,395 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/43d8b8753bd54cc3b442bdc6adfd0470 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/43d8b8753bd54cc3b442bdc6adfd0470 2024-11-14T07:25:06,402 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:06,403 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/2f96721c152446409d05df8d3ffedbd0 is 50, key is test_row_0/C:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,410 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 43d8b8753bd54cc3b442bdc6adfd0470(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:06,410 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,410 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=13, startTime=1731569106353; duration=0sec 2024-11-14T07:25:06,410 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:06,410 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:06,410 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:06,412 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:06,412 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:06,412 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:06,412 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c6cfea64daa74fd3b2bb32fb38c7befe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=37.1 K 2024-11-14T07:25:06,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6cfea64daa74fd3b2bb32fb38c7befe, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731569104215 2024-11-14T07:25:06,414 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd0f451687745159a23eb78918fe8b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731569104863 2024-11-14T07:25:06,414 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ab3b8a6eb14f95b788eaa8b811f91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:06,426 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:06,426 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/69377f492bef42e89080187a949de998 is 50, key is test_row_0/B:col10/1731569106080/Put/seqid=0 2024-11-14T07:25:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741939_1115 (size=13425) 2024-11-14T07:25:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741940_1116 (size=13459) 2024-11-14T07:25:06,485 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/69377f492bef42e89080187a949de998 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/69377f492bef42e89080187a949de998 2024-11-14T07:25:06,494 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 69377f492bef42e89080187a949de998(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:06,495 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,495 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=13, startTime=1731569106356; duration=0sec 2024-11-14T07:25:06,495 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:06,495 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:06,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:06,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:06,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/36d3f429c3d64e1baae539ad21c5ea69 is 50, key is test_row_0/A:col10/1731569106531/Put/seqid=0 2024-11-14T07:25:06,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741941_1117 (size=12301) 2024-11-14T07:25:06,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/36d3f429c3d64e1baae539ad21c5ea69 2024-11-14T07:25:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/20e835cb12fc4c10b4df34278421156f is 50, key is test_row_0/B:col10/1731569106531/Put/seqid=0 2024-11-14T07:25:06,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741942_1118 (size=12301) 2024-11-14T07:25:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,853 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/2f96721c152446409d05df8d3ffedbd0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2f96721c152446409d05df8d3ffedbd0 2024-11-14T07:25:06,858 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 2f96721c152446409d05df8d3ffedbd0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:06,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:06,858 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=13, startTime=1731569106356; duration=0sec 2024-11-14T07:25:06,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:06,858 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569166905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569166905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569166905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569166905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569166905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:06,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T07:25:06,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:06,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T07:25:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:06,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:06,990 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:06,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:07,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/20e835cb12fc4c10b4df34278421156f 2024-11-14T07:25:07,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/8ad5c49e4c27425b9568437383826383 is 50, key is test_row_0/C:col10/1731569106531/Put/seqid=0 2024-11-14T07:25:07,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741943_1119 (size=12301) 2024-11-14T07:25:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:07,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:25:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569167215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569167215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569167215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569167215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569167215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:25:07,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:07,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:07,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/8ad5c49e4c27425b9568437383826383 2024-11-14T07:25:07,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/36d3f429c3d64e1baae539ad21c5ea69 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69 2024-11-14T07:25:07,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T07:25:07,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:25:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/20e835cb12fc4c10b4df34278421156f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f 2024-11-14T07:25:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:07,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T07:25:07,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/8ad5c49e4c27425b9568437383826383 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383 2024-11-14T07:25:07,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383, entries=150, sequenceid=500, filesize=12.0 K 2024-11-14T07:25:07,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 935ms, sequenceid=500, compaction requested=false 2024-11-14T07:25:07,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:07,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T07:25:07,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:07,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:25:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/188be8b8310d498792af81dc20d45afb is 50, key is test_row_0/A:col10/1731569106580/Put/seqid=0 2024-11-14T07:25:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741944_1120 (size=12301) 2024-11-14T07:25:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:07,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569167731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569167732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569167734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569167735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569167735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569167845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569167845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569167845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569167846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569167846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/188be8b8310d498792af81dc20d45afb 2024-11-14T07:25:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569168054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569168055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569168055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569168055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569168055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/715339d529b64839a660de9b362ed1ff is 50, key is test_row_0/B:col10/1731569106580/Put/seqid=0 2024-11-14T07:25:08,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741945_1121 (size=12301) 2024-11-14T07:25:08,104 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/715339d529b64839a660de9b362ed1ff 2024-11-14T07:25:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/07ac20b507d444909e5509cc22599f67 is 50, key is test_row_0/C:col10/1731569106580/Put/seqid=0 2024-11-14T07:25:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:08,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741946_1122 (size=12301) 2024-11-14T07:25:08,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/07ac20b507d444909e5509cc22599f67 2024-11-14T07:25:08,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/188be8b8310d498792af81dc20d45afb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb 2024-11-14T07:25:08,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb, entries=150, sequenceid=526, filesize=12.0 K 2024-11-14T07:25:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/715339d529b64839a660de9b362ed1ff as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff 2024-11-14T07:25:08,175 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff, entries=150, sequenceid=526, filesize=12.0 K 2024-11-14T07:25:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/07ac20b507d444909e5509cc22599f67 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67 2024-11-14T07:25:08,206 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67, entries=150, sequenceid=526, filesize=12.0 K 2024-11-14T07:25:08,207 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 604ms, sequenceid=526, compaction requested=true 2024-11-14T07:25:08,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:08,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:08,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T07:25:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T07:25:08,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T07:25:08,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2190 sec 2024-11-14T07:25:08,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2280 sec 2024-11-14T07:25:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:08,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:08,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:25:08,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/afadb1bf1c7541469b656eafad1be580 is 50, key is test_row_0/A:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:08,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569168404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569168404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569168406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569168409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569168412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741947_1123 (size=12301) 2024-11-14T07:25:08,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/afadb1bf1c7541469b656eafad1be580 2024-11-14T07:25:08,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/2e2b7197f2124766ba33ec6834bb1288 is 50, key is test_row_0/B:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:08,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741948_1124 (size=12301) 2024-11-14T07:25:08,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/2e2b7197f2124766ba33ec6834bb1288 2024-11-14T07:25:08,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f5f89d05c14498798c7546edda292e1 is 50, key is test_row_0/C:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:08,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741949_1125 (size=12301) 2024-11-14T07:25:08,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f5f89d05c14498798c7546edda292e1 2024-11-14T07:25:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569168514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569168514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569168514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569168525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569168526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/afadb1bf1c7541469b656eafad1be580 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580 2024-11-14T07:25:08,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T07:25:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/2e2b7197f2124766ba33ec6834bb1288 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288 2024-11-14T07:25:08,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T07:25:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/5f5f89d05c14498798c7546edda292e1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1 2024-11-14T07:25:08,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1, entries=150, sequenceid=539, filesize=12.0 K 2024-11-14T07:25:08,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 198ms, sequenceid=539, compaction requested=true 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:08,568 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:08,568 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:08,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:08,570 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:08,570 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:08,570 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:08,570 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:08,570 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:08,570 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:08,570 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/43d8b8753bd54cc3b442bdc6adfd0470, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=49.2 K 2024-11-14T07:25:08,571 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/69377f492bef42e89080187a949de998, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=49.2 K 2024-11-14T07:25:08,571 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d8b8753bd54cc3b442bdc6adfd0470, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:08,571 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 69377f492bef42e89080187a949de998, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:08,572 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d3f429c3d64e1baae539ad21c5ea69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731569106527 2024-11-14T07:25:08,572 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e835cb12fc4c10b4df34278421156f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731569106527 2024-11-14T07:25:08,572 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188be8b8310d498792af81dc20d45afb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731569106580 2024-11-14T07:25:08,572 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 715339d529b64839a660de9b362ed1ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731569106580 2024-11-14T07:25:08,573 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2b7197f2124766ba33ec6834bb1288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:08,573 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting afadb1bf1c7541469b656eafad1be580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:08,604 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:08,604 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/601bd9ce5bcb4bc68dba92a5a192c56c is 50, key is test_row_0/B:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:08,606 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:08,607 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/9d593f2f12644a3d9aa7f3310e19b4d0 is 50, key is test_row_0/A:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:08,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741951_1127 (size=13595) 2024-11-14T07:25:08,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741950_1126 (size=13595) 2024-11-14T07:25:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:08,728 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:08,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:25:08,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/9eb0a6d837164bcc9a52e111e0fd2f89 is 50, key is test_row_0/A:col10/1731569108398/Put/seqid=0 2024-11-14T07:25:08,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741952_1128 (size=17181) 2024-11-14T07:25:08,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/9eb0a6d837164bcc9a52e111e0fd2f89 2024-11-14T07:25:08,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/9c3f49a016aa4ad495582bd039990271 is 50, key is test_row_0/B:col10/1731569108398/Put/seqid=0 2024-11-14T07:25:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741953_1129 (size=12301) 2024-11-14T07:25:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569168742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569168744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569168780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569168784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569168780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569168895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569168895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569168904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569168905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,051 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/601bd9ce5bcb4bc68dba92a5a192c56c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/601bd9ce5bcb4bc68dba92a5a192c56c 2024-11-14T07:25:09,055 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/9d593f2f12644a3d9aa7f3310e19b4d0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9d593f2f12644a3d9aa7f3310e19b4d0 2024-11-14T07:25:09,062 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into 601bd9ce5bcb4bc68dba92a5a192c56c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:09,062 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:09,062 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569108568; duration=0sec 2024-11-14T07:25:09,062 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:09,062 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:09,063 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 9d593f2f12644a3d9aa7f3310e19b4d0(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:09,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:09,064 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569108568; duration=0sec 2024-11-14T07:25:09,064 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:09,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:09,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:09,065 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:09,065 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:09,065 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,065 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2f96721c152446409d05df8d3ffedbd0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=49.1 K 2024-11-14T07:25:09,066 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f96721c152446409d05df8d3ffedbd0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731569106067 2024-11-14T07:25:09,066 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad5c49e4c27425b9568437383826383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1731569106527 2024-11-14T07:25:09,067 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ac20b507d444909e5509cc22599f67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1731569106580 2024-11-14T07:25:09,068 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5f89d05c14498798c7546edda292e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:09,078 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:09,079 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/2a8e19d18305447ba46f05faef7cd314 is 50, key is test_row_0/C:col10/1731569107733/Put/seqid=0 2024-11-14T07:25:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569169084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741954_1130 (size=13561) 2024-11-14T07:25:09,097 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/2a8e19d18305447ba46f05faef7cd314 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2a8e19d18305447ba46f05faef7cd314 2024-11-14T07:25:09,104 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into 2a8e19d18305447ba46f05faef7cd314(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:09,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:09,104 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569108568; duration=0sec 2024-11-14T07:25:09,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:09,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569169105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569169105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569169114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569169114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T07:25:09,135 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:09,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-14T07:25:09,137 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:25:09,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:09,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:09,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/9c3f49a016aa4ad495582bd039990271 2024-11-14T07:25:09,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/301cf57ebe3a40cca2344d106c73f86b is 50, key is test_row_0/C:col10/1731569108398/Put/seqid=0 2024-11-14T07:25:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741955_1131 (size=12301) 2024-11-14T07:25:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:25:09,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:25:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569169415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569169415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569169424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569169424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:25:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:25:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569169594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:25:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:09,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/301cf57ebe3a40cca2344d106c73f86b 2024-11-14T07:25:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/9eb0a6d837164bcc9a52e111e0fd2f89 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89 2024-11-14T07:25:09,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89, entries=250, sequenceid=564, filesize=16.8 K 2024-11-14T07:25:09,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/9c3f49a016aa4ad495582bd039990271 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271 2024-11-14T07:25:09,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T07:25:09,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/301cf57ebe3a40cca2344d106c73f86b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b 2024-11-14T07:25:09,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T07:25:09,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 898ms, sequenceid=564, compaction requested=false 2024-11-14T07:25:09,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:09,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T07:25:09,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:09,751 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:25:09,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f4f9b27fca9e4a10b167e5baf2d34db3 is 50, key is test_row_0/A:col10/1731569108740/Put/seqid=0 2024-11-14T07:25:09,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741956_1132 (size=12301) 2024-11-14T07:25:09,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f4f9b27fca9e4a10b167e5baf2d34db3 2024-11-14T07:25:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:25:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1916f672f4a84a07930ea23e0df8f9ca is 50, key is test_row_0/B:col10/1731569108740/Put/seqid=0 2024-11-14T07:25:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741957_1133 (size=12301) 2024-11-14T07:25:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:09,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569169950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569169950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569169951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569169951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569170078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569170078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569170078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569170078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1916f672f4a84a07930ea23e0df8f9ca 2024-11-14T07:25:10,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/92b9d159d02442b39dfaad841ac1fe13 is 50, key is test_row_0/C:col10/1731569108740/Put/seqid=0 2024-11-14T07:25:10,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741958_1134 (size=12301) 2024-11-14T07:25:10,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/92b9d159d02442b39dfaad841ac1fe13 2024-11-14T07:25:10,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/f4f9b27fca9e4a10b167e5baf2d34db3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3 2024-11-14T07:25:10,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3, entries=150, sequenceid=579, filesize=12.0 K 2024-11-14T07:25:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/1916f672f4a84a07930ea23e0df8f9ca as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca 2024-11-14T07:25:10,233 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca, entries=150, sequenceid=579, filesize=12.0 K 2024-11-14T07:25:10,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/92b9d159d02442b39dfaad841ac1fe13 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13 2024-11-14T07:25:10,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13, entries=150, sequenceid=579, filesize=12.0 K 2024-11-14T07:25:10,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 493ms, sequenceid=579, compaction requested=true 2024-11-14T07:25:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-14T07:25:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-14T07:25:10,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T07:25:10,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-14T07:25:10,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1190 sec 2024-11-14T07:25:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T07:25:10,275 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:10,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-14T07:25:10,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:10,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:10,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:10,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:10,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:25:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:10,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ba599d56b8b41cdaa4a1943b29b5c51 is 50, key is test_row_0/A:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:10,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741959_1135 (size=14741) 2024-11-14T07:25:10,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569170324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569170331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569170331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569170331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:10,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:10,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569170435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569170445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569170445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569170445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:10,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731569170605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,606 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in eb8aa6a5cb82db71eb9aa5187f031ed4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569170644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569170654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569170654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569170655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ba599d56b8b41cdaa4a1943b29b5c51 2024-11-14T07:25:10,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/3741730875804894ac4e3e766f521d6e is 50, key is test_row_0/B:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:10,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741960_1136 (size=12301) 2024-11-14T07:25:10,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:10,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569170954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569170965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569170964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569170965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:11,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:11,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/3741730875804894ac4e3e766f521d6e 2024-11-14T07:25:11,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/bd49c511ca53483b8508e51201b7b0e5 is 50, key is test_row_0/C:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:11,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741961_1137 (size=12301) 2024-11-14T07:25:11,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:11,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36296 deadline: 1731569171464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36318 deadline: 1731569171475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731569171475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36334 deadline: 1731569171475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:11,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. as already flushing 2024-11-14T07:25:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:11,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/bd49c511ca53483b8508e51201b7b0e5 2024-11-14T07:25:11,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/6ba599d56b8b41cdaa4a1943b29b5c51 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51 2024-11-14T07:25:11,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51, entries=200, sequenceid=606, filesize=14.4 K 2024-11-14T07:25:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/3741730875804894ac4e3e766f521d6e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e 2024-11-14T07:25:11,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e, entries=150, sequenceid=606, filesize=12.0 K 2024-11-14T07:25:11,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/bd49c511ca53483b8508e51201b7b0e5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5 2024-11-14T07:25:11,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5, entries=150, sequenceid=606, filesize=12.0 K 2024-11-14T07:25:11,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 1266ms, sequenceid=606, compaction requested=true 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:11,556 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:11,556 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eb8aa6a5cb82db71eb9aa5187f031ed4:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:11,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:11,557 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:11,557 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/B is initiating minor compaction (all files) 2024-11-14T07:25:11,557 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/B in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,558 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/601bd9ce5bcb4bc68dba92a5a192c56c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=49.3 K 2024-11-14T07:25:11,558 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:11,558 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/A is initiating minor compaction (all files) 2024-11-14T07:25:11,558 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/A in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,558 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9d593f2f12644a3d9aa7f3310e19b4d0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=56.5 K 2024-11-14T07:25:11,558 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 601bd9ce5bcb4bc68dba92a5a192c56c, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d593f2f12644a3d9aa7f3310e19b4d0, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3f49a016aa4ad495582bd039990271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731569108398 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb0a6d837164bcc9a52e111e0fd2f89, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731569108398 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1916f672f4a84a07930ea23e0df8f9ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731569108740 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f9b27fca9e4a10b167e5baf2d34db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731569108740 2024-11-14T07:25:11,559 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3741730875804894ac4e3e766f521d6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731569109943 2024-11-14T07:25:11,560 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba599d56b8b41cdaa4a1943b29b5c51, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731569109943 2024-11-14T07:25:11,569 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#A#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:11,569 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:11,569 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/3cc0f02d061d47b7b3f0da89043bfc93 is 50, key is test_row_0/A:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:11,570 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/f0002035838746d780b3fbd2467d9764 is 50, key is test_row_0/B:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:11,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741962_1138 (size=13731) 2024-11-14T07:25:11,579 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/3cc0f02d061d47b7b3f0da89043bfc93 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/3cc0f02d061d47b7b3f0da89043bfc93 2024-11-14T07:25:11,588 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/A of eb8aa6a5cb82db71eb9aa5187f031ed4 into 3cc0f02d061d47b7b3f0da89043bfc93(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:11,588 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:11,588 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/A, priority=12, startTime=1731569111556; duration=0sec 2024-11-14T07:25:11,588 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:11,588 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:A 2024-11-14T07:25:11,588 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:11,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741963_1139 (size=13731) 2024-11-14T07:25:11,592 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:11,592 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): eb8aa6a5cb82db71eb9aa5187f031ed4/C is initiating minor compaction (all files) 2024-11-14T07:25:11,592 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eb8aa6a5cb82db71eb9aa5187f031ed4/C in TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,592 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2a8e19d18305447ba46f05faef7cd314, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp, totalSize=49.3 K 2024-11-14T07:25:11,593 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a8e19d18305447ba46f05faef7cd314, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1731569107730 2024-11-14T07:25:11,594 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301cf57ebe3a40cca2344d106c73f86b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731569108398 2024-11-14T07:25:11,594 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b9d159d02442b39dfaad841ac1fe13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1731569108740 2024-11-14T07:25:11,595 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd49c511ca53483b8508e51201b7b0e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731569109943 2024-11-14T07:25:11,600 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/f0002035838746d780b3fbd2467d9764 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f0002035838746d780b3fbd2467d9764 2024-11-14T07:25:11,610 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/B of eb8aa6a5cb82db71eb9aa5187f031ed4 into f0002035838746d780b3fbd2467d9764(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:11,610 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:11,610 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/B, priority=12, startTime=1731569111556; duration=0sec 2024-11-14T07:25:11,610 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:11,611 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:B 2024-11-14T07:25:11,613 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eb8aa6a5cb82db71eb9aa5187f031ed4#C#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:11,614 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/ba171db053634e5d97899332d36c3b3c is 50, key is test_row_0/C:col10/1731569110288/Put/seqid=0 2024-11-14T07:25:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741964_1140 (size=13697) 2024-11-14T07:25:11,638 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/ba171db053634e5d97899332d36c3b3c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/ba171db053634e5d97899332d36c3b3c 2024-11-14T07:25:11,649 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eb8aa6a5cb82db71eb9aa5187f031ed4/C of eb8aa6a5cb82db71eb9aa5187f031ed4 into ba171db053634e5d97899332d36c3b3c(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:11,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:11,649 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., storeName=eb8aa6a5cb82db71eb9aa5187f031ed4/C, priority=12, startTime=1731569111556; duration=0sec 2024-11-14T07:25:11,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:11,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eb8aa6a5cb82db71eb9aa5187f031ed4:C 2024-11-14T07:25:11,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-14T07:25:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:11,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T07:25:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7ec000497c8948acb254c35ea59fd25a is 50, key is test_row_0/A:col10/1731569110329/Put/seqid=0 2024-11-14T07:25:11,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741965_1141 (size=12301) 2024-11-14T07:25:11,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7ec000497c8948acb254c35ea59fd25a 2024-11-14T07:25:11,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/df64867d4a7f444582517bd217516bfd is 50, key is test_row_0/B:col10/1731569110329/Put/seqid=0 2024-11-14T07:25:11,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741966_1142 (size=12301) 2024-11-14T07:25:11,711 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/df64867d4a7f444582517bd217516bfd 2024-11-14T07:25:11,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/19ec5603437540419aa48dc2be0fc4f6 is 50, key is test_row_0/C:col10/1731569110329/Put/seqid=0 2024-11-14T07:25:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741967_1143 (size=12301) 2024-11-14T07:25:12,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/19ec5603437540419aa48dc2be0fc4f6 2024-11-14T07:25:12,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/7ec000497c8948acb254c35ea59fd25a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7ec000497c8948acb254c35ea59fd25a 2024-11-14T07:25:12,199 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7ec000497c8948acb254c35ea59fd25a, entries=150, sequenceid=619, filesize=12.0 K 2024-11-14T07:25:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/df64867d4a7f444582517bd217516bfd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/df64867d4a7f444582517bd217516bfd 2024-11-14T07:25:12,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/df64867d4a7f444582517bd217516bfd, entries=150, sequenceid=619, filesize=12.0 K 2024-11-14T07:25:12,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-14T07:25:12,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/19ec5603437540419aa48dc2be0fc4f6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/19ec5603437540419aa48dc2be0fc4f6 2024-11-14T07:25:12,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/19ec5603437540419aa48dc2be0fc4f6, entries=150, sequenceid=619, filesize=12.0 K 2024-11-14T07:25:12,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 555ms, sequenceid=619, compaction requested=false 2024-11-14T07:25:12,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for eb8aa6a5cb82db71eb9aa5187f031ed4: 2024-11-14T07:25:12,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:12,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-14T07:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-14T07:25:12,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T07:25:12,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9360 sec 2024-11-14T07:25:12,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9410 sec 2024-11-14T07:25:12,269 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T07:25:12,269 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,269 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T07:25:12,269 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,269 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T07:25:12,269 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T07:25:12,269 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,269 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,273 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,273 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,273 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,273 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,273 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,274 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,274 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,274 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T07:25:12,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:12,488 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T07:25:12,489 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,489 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,489 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,496 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T07:25:12,496 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T07:25:12,496 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,496 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,496 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,496 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,496 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,496 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,497 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T07:25:12,497 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,497 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,497 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,617 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T07:25:12,617 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:12,617 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,617 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-14T07:25:12,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3661 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3777 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1743 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5223 rows 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1741 2024-11-14T07:25:12,619 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5220 rows 2024-11-14T07:25:12,619 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:25:12,620 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:25:12,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:12,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:12,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:25:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:12,631 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569112631"}]},"ts":"1731569112631"} 2024-11-14T07:25:12,633 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:25:12,633 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:25:12,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:25:12,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, UNASSIGN}] 2024-11-14T07:25:12,640 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, UNASSIGN 2024-11-14T07:25:12,642 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=eb8aa6a5cb82db71eb9aa5187f031ed4, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:12,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, UNASSIGN because future has completed 2024-11-14T07:25:12,645 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:25:12,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:12,805 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:12,805 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:25:12,806 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing eb8aa6a5cb82db71eb9aa5187f031ed4, disabling compactions & flushes 2024-11-14T07:25:12,806 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:12,806 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:12,806 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. after waiting 0 ms 2024-11-14T07:25:12,806 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:12,807 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing eb8aa6a5cb82db71eb9aa5187f031ed4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T07:25:12,812 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/974706908f65434ca5bd641d7e84233e is 50, key is test_row_0/A:col10/1731569112495/Put/seqid=0 2024-11-14T07:25:12,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741968_1144 (size=12301) 2024-11-14T07:25:12,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:13,217 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/974706908f65434ca5bd641d7e84233e 2024-11-14T07:25:13,229 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/d862c9cf06fc409ab8f4d2d1a15a2c71 is 50, key is test_row_0/B:col10/1731569112495/Put/seqid=0 2024-11-14T07:25:13,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741969_1145 (size=12301) 2024-11-14T07:25:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:13,634 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/d862c9cf06fc409ab8f4d2d1a15a2c71 2024-11-14T07:25:13,649 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7f2f402c82e3401883af9f425112745b is 50, key is test_row_0/C:col10/1731569112495/Put/seqid=0 2024-11-14T07:25:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741970_1146 (size=12301) 2024-11-14T07:25:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:13,896 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:25:14,055 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7f2f402c82e3401883af9f425112745b 2024-11-14T07:25:14,069 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/A/974706908f65434ca5bd641d7e84233e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/974706908f65434ca5bd641d7e84233e 2024-11-14T07:25:14,076 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/974706908f65434ca5bd641d7e84233e, entries=150, sequenceid=627, filesize=12.0 K 2024-11-14T07:25:14,078 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/B/d862c9cf06fc409ab8f4d2d1a15a2c71 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/d862c9cf06fc409ab8f4d2d1a15a2c71 2024-11-14T07:25:14,083 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/d862c9cf06fc409ab8f4d2d1a15a2c71, entries=150, sequenceid=627, filesize=12.0 K 2024-11-14T07:25:14,085 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/.tmp/C/7f2f402c82e3401883af9f425112745b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7f2f402c82e3401883af9f425112745b 2024-11-14T07:25:14,091 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7f2f402c82e3401883af9f425112745b, entries=150, sequenceid=627, filesize=12.0 K 2024-11-14T07:25:14,092 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 1286ms, sequenceid=627, compaction requested=true 2024-11-14T07:25:14,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f498efc2fad54f23932310faf49527fa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/90c35a71ee954c44b3a67424fc695dcc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b40f46422af04fc8a31db15c0718b977, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/53cc7b8dca9b40c4b982737b1f3d5d69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/82a6bf89d85a4453bed158733d984985, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/72bdd8a5d74f4ad38a688ec75697907f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/05af92a78a4e4eb0b7fd0b6822d91147, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/29ceb3c9498346989f1d78816c7b07db, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f254ee9c45bc4867afd92e0767081e7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/43d8b8753bd54cc3b442bdc6adfd0470, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9d593f2f12644a3d9aa7f3310e19b4d0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51] to archive 2024-11-14T07:25:14,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:14,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d91cfb834d4c4933bdd07a41460c86b1 2024-11-14T07:25:14,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/853f4a68f34f4d7ca9455f4df52945e0 2024-11-14T07:25:14,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/71e3115fd00f46579c76036a4251eb27 2024-11-14T07:25:14,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b056e19e1a27404285dc7f3bb6cf4fa0 2024-11-14T07:25:14,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f498efc2fad54f23932310faf49527fa to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f498efc2fad54f23932310faf49527fa 2024-11-14T07:25:14,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/fa42b88c42644e30b491b46594b10404 2024-11-14T07:25:14,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a8ced28e44064049a61e3c8325dbadd4 2024-11-14T07:25:14,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ad99620ff1044c47a1aacc77ec7c8a12 2024-11-14T07:25:14,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/90c35a71ee954c44b3a67424fc695dcc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/90c35a71ee954c44b3a67424fc695dcc 2024-11-14T07:25:14,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/d1dfb8f1b9d04d8295b6c8c648124d09 2024-11-14T07:25:14,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b40f46422af04fc8a31db15c0718b977 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b40f46422af04fc8a31db15c0718b977 2024-11-14T07:25:14,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6f6932416eff47f4ac5604b80b284d03 2024-11-14T07:25:14,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7237b7b85af44745940f4aef3f613282 2024-11-14T07:25:14,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/53cc7b8dca9b40c4b982737b1f3d5d69 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/53cc7b8dca9b40c4b982737b1f3d5d69 2024-11-14T07:25:14,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ecfe50913304953a91bf1f15919614a 2024-11-14T07:25:14,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/197f049c799a4b70894bddf57eb5bf36 2024-11-14T07:25:14,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/8eedaeeb2d154d4db1c590d9184b804b 2024-11-14T07:25:14,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/cd995180ed4b4e01a59246fcad2f49bd 2024-11-14T07:25:14,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/82a6bf89d85a4453bed158733d984985 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/82a6bf89d85a4453bed158733d984985 2024-11-14T07:25:14,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/0c2cdde8f872422f9027450f7dc86eda 2024-11-14T07:25:14,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/72bdd8a5d74f4ad38a688ec75697907f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/72bdd8a5d74f4ad38a688ec75697907f 2024-11-14T07:25:14,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c1a6344e47d34e40a723dea9854f63e3 2024-11-14T07:25:14,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/68ddccede5324f369922bab97084075c 2024-11-14T07:25:14,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/05af92a78a4e4eb0b7fd0b6822d91147 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/05af92a78a4e4eb0b7fd0b6822d91147 2024-11-14T07:25:14,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b46100911f754d938624d88cf954075f 2024-11-14T07:25:14,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/ebcc8f978d8f46c194b10a25f782f023 2024-11-14T07:25:14,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/4f87661c09de41a4a1382d96e63685f8 2024-11-14T07:25:14,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/b1e8afa6fb5f4bf7a04ef1440072e1a2 2024-11-14T07:25:14,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/29ceb3c9498346989f1d78816c7b07db to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/29ceb3c9498346989f1d78816c7b07db 2024-11-14T07:25:14,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/a2074183a6fb4d0bac24ee4b55cbb3c1 2024-11-14T07:25:14,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/c42c70af168548258ff89d1534a0f9d0 2024-11-14T07:25:14,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f254ee9c45bc4867afd92e0767081e7c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f254ee9c45bc4867afd92e0767081e7c 2024-11-14T07:25:14,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/5a820422e4994e94bb565eae2ff6759f 2024-11-14T07:25:14,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/43d8b8753bd54cc3b442bdc6adfd0470 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/43d8b8753bd54cc3b442bdc6adfd0470 2024-11-14T07:25:14,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/24eda2f8f5ba430895120ec6bd1ef164 2024-11-14T07:25:14,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/36d3f429c3d64e1baae539ad21c5ea69 2024-11-14T07:25:14,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/188be8b8310d498792af81dc20d45afb 2024-11-14T07:25:14,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9d593f2f12644a3d9aa7f3310e19b4d0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9d593f2f12644a3d9aa7f3310e19b4d0 2024-11-14T07:25:14,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/afadb1bf1c7541469b656eafad1be580 2024-11-14T07:25:14,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/9eb0a6d837164bcc9a52e111e0fd2f89 2024-11-14T07:25:14,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/f4f9b27fca9e4a10b167e5baf2d34db3 2024-11-14T07:25:14,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/6ba599d56b8b41cdaa4a1943b29b5c51 2024-11-14T07:25:14,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/5c8d278566164e3590e4f6462889f5da, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/91ef286135b34726b6c8d170e3cbc4fb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1b0710f2587c42a1b0c1ee4cd6760490, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4f3c87d9e99d459e9522db738935ea2a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b483ea60fe214e24841ac5deb34ab001, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/abeb284ee53b421d9256ca38b2b6ff64, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1e476785c642424182a68631f23e6bf3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1c3445ebf3a24949a26fdfcfbc47f82e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c6cfea64daa74fd3b2bb32fb38c7befe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/69377f492bef42e89080187a949de998, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/601bd9ce5bcb4bc68dba92a5a192c56c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e] to archive 2024-11-14T07:25:14,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:14,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e6dc5e1d1cb44227a5a4bbc3d6ce4d8a 2024-11-14T07:25:14,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/ac365b78798a4db5b7885a8d81eaa548 2024-11-14T07:25:14,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e0cb76febf334e719d76ef0e78bdc7b6 2024-11-14T07:25:14,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/5c8d278566164e3590e4f6462889f5da to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/5c8d278566164e3590e4f6462889f5da 2024-11-14T07:25:14,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e2307ae1817d4c6986867f82f63f2172 2024-11-14T07:25:14,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/e03bf39193e446539406411ac44c5d78 2024-11-14T07:25:14,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/61176a38bc90450abb375b6d8c76a4d8 2024-11-14T07:25:14,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/91ef286135b34726b6c8d170e3cbc4fb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/91ef286135b34726b6c8d170e3cbc4fb 2024-11-14T07:25:14,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f167303a57b54deb92d984d83a47b65b 2024-11-14T07:25:14,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/34b33a022a0e4642a7d3647805700f59 2024-11-14T07:25:14,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1b0710f2587c42a1b0c1ee4cd6760490 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1b0710f2587c42a1b0c1ee4cd6760490 2024-11-14T07:25:14,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b8df62cf85334eefa9c9bf85544c27b6 2024-11-14T07:25:14,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bb0b253748da45f08c98f46642d98fe6 2024-11-14T07:25:14,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4f3c87d9e99d459e9522db738935ea2a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4f3c87d9e99d459e9522db738935ea2a 2024-11-14T07:25:14,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/64d554dc62664558908162f4716db5a6 2024-11-14T07:25:14,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/4a26ef9d15414dc999c45d5749f0e7a0 2024-11-14T07:25:14,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c07248a9691e4fe8875a2048fe8875c1 2024-11-14T07:25:14,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b483ea60fe214e24841ac5deb34ab001 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/b483ea60fe214e24841ac5deb34ab001 2024-11-14T07:25:14,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8b67354a7ee64de7bd19d37b43194791 2024-11-14T07:25:14,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/8be62445acf24666adff2519ab54a394 2024-11-14T07:25:14,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/abeb284ee53b421d9256ca38b2b6ff64 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/abeb284ee53b421d9256ca38b2b6ff64 2024-11-14T07:25:14,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/0cd46a1eb2e847f4a3eac5c06f01b4d1 2024-11-14T07:25:14,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/042f935afff94336a256b2eb90c15af8 2024-11-14T07:25:14,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1e476785c642424182a68631f23e6bf3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1e476785c642424182a68631f23e6bf3 2024-11-14T07:25:14,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/14438bcf6e3a41018c737b9c6d43fad0 2024-11-14T07:25:14,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/bbd8d470b14d47b2b046f2db22cdafc2 2024-11-14T07:25:14,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1c3445ebf3a24949a26fdfcfbc47f82e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1c3445ebf3a24949a26fdfcfbc47f82e 2024-11-14T07:25:14,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/fe299a363aaf4c65aefba109e1aa452b 2024-11-14T07:25:14,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/534297c24138478fa9c4247b50f139fe 2024-11-14T07:25:14,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/25dc9f9e5bfe41a68c8d6e65004568e9 2024-11-14T07:25:14,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c6cfea64daa74fd3b2bb32fb38c7befe to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/c6cfea64daa74fd3b2bb32fb38c7befe 2024-11-14T07:25:14,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/62ab7917cc29436682f8d9de8137ec6b 2024-11-14T07:25:14,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/afd0f451687745159a23eb78918fe8b1 2024-11-14T07:25:14,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/69377f492bef42e89080187a949de998 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/69377f492bef42e89080187a949de998 2024-11-14T07:25:14,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/a0ab3b8a6eb14f95b788eaa8b811f91e 2024-11-14T07:25:14,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/20e835cb12fc4c10b4df34278421156f 2024-11-14T07:25:14,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/715339d529b64839a660de9b362ed1ff 2024-11-14T07:25:14,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/601bd9ce5bcb4bc68dba92a5a192c56c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/601bd9ce5bcb4bc68dba92a5a192c56c 2024-11-14T07:25:14,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/2e2b7197f2124766ba33ec6834bb1288 2024-11-14T07:25:14,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/9c3f49a016aa4ad495582bd039990271 2024-11-14T07:25:14,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/1916f672f4a84a07930ea23e0df8f9ca 2024-11-14T07:25:14,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/3741730875804894ac4e3e766f521d6e 2024-11-14T07:25:14,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8fd68a53fe414765a10f733337127d47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d445624c6ec749b5a17fb2237d00b28f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/701aa74c50b2442f803d0eb287006acb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/3675a89b60ee453ebecaab34007dfc6f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/def8ee88074c469cb6cdc73121ce1dff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4ac95cfa900c49c0b481e7ab480b4b6c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6adf1f50b7d642829a863b70be306cab, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/84d82f1036cb4df584ca6977f4d3ce14, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2f96721c152446409d05df8d3ffedbd0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2a8e19d18305447ba46f05faef7cd314, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5] to archive 2024-11-14T07:25:14,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:14,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/35f32ac8857f41588ea1ae6f0e538b2b 2024-11-14T07:25:14,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/dc190a87634f410f8ae79af18b24b209 2024-11-14T07:25:14,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/656d156722734553ab68e9c381b1b48e 2024-11-14T07:25:14,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8fd68a53fe414765a10f733337127d47 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8fd68a53fe414765a10f733337127d47 2024-11-14T07:25:14,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1778951427d74dcba83d99d77232f5a4 2024-11-14T07:25:14,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/18e20658c7e94cf8b9846a244159381d 2024-11-14T07:25:14,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d386ab692fe840ef87434aa2d3f49a8b 2024-11-14T07:25:14,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d445624c6ec749b5a17fb2237d00b28f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/d445624c6ec749b5a17fb2237d00b28f 2024-11-14T07:25:14,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/e96f3936b89e445e911cd7a1322aa3dd 2024-11-14T07:25:14,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/da1e039009244ecfbe0d579b24cb7900 2024-11-14T07:25:14,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/701aa74c50b2442f803d0eb287006acb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/701aa74c50b2442f803d0eb287006acb 2024-11-14T07:25:14,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/313d4a5066ea47f1a173d483870c83bc 2024-11-14T07:25:14,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fda15365a4dd44f4990c93ef27cccdc6 2024-11-14T07:25:14,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/3675a89b60ee453ebecaab34007dfc6f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/3675a89b60ee453ebecaab34007dfc6f 2024-11-14T07:25:14,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/9db390f72a59497f98386cb54dcba34c 2024-11-14T07:25:14,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/f1cc543399574868a41d2993d2925c44 2024-11-14T07:25:14,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a1ae5bfd3a2449798cc21f891b06b85b 2024-11-14T07:25:14,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/def8ee88074c469cb6cdc73121ce1dff to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/def8ee88074c469cb6cdc73121ce1dff 2024-11-14T07:25:14,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f7f1c7dd7a74e75a3c35291ac137be0 2024-11-14T07:25:14,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7c8f1e807c7c43d69f316715e27b8645 2024-11-14T07:25:14,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4ac95cfa900c49c0b481e7ab480b4b6c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4ac95cfa900c49c0b481e7ab480b4b6c 2024-11-14T07:25:14,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6eb8c9b93b6e4a6c9abae35e699cb623 2024-11-14T07:25:14,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5521d3b7aad445d2b46d55a83ae78006 2024-11-14T07:25:14,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/fc723fe6ef374e9097b48039a34b6674 2024-11-14T07:25:14,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/20c1939326144493b221039c5edde33c 2024-11-14T07:25:14,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6adf1f50b7d642829a863b70be306cab to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/6adf1f50b7d642829a863b70be306cab 2024-11-14T07:25:14,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5c206d5350f1446b9675f4b70a9400a0 2024-11-14T07:25:14,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/1efa0afc280843959aa1765192517f0e 2024-11-14T07:25:14,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a76db206f30f481d9a3031ca08614c2f 2024-11-14T07:25:14,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/84d82f1036cb4df584ca6977f4d3ce14 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/84d82f1036cb4df584ca6977f4d3ce14 2024-11-14T07:25:14,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/526a4f9a3c0d43cfbf41246b230ff7cd 2024-11-14T07:25:14,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/4da038e6e292439d8829a59cb3af64fd 2024-11-14T07:25:14,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2f96721c152446409d05df8d3ffedbd0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2f96721c152446409d05df8d3ffedbd0 2024-11-14T07:25:14,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/a560164161f24101857379e2dc2035c0 2024-11-14T07:25:14,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/8ad5c49e4c27425b9568437383826383 2024-11-14T07:25:14,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/07ac20b507d444909e5509cc22599f67 2024-11-14T07:25:14,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2a8e19d18305447ba46f05faef7cd314 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/2a8e19d18305447ba46f05faef7cd314 2024-11-14T07:25:14,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/5f5f89d05c14498798c7546edda292e1 2024-11-14T07:25:14,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/301cf57ebe3a40cca2344d106c73f86b 2024-11-14T07:25:14,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/92b9d159d02442b39dfaad841ac1fe13 2024-11-14T07:25:14,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/bd49c511ca53483b8508e51201b7b0e5 2024-11-14T07:25:14,300 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/recovered.edits/630.seqid, newMaxSeqId=630, maxSeqId=1 2024-11-14T07:25:14,301 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. 2024-11-14T07:25:14,301 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for eb8aa6a5cb82db71eb9aa5187f031ed4: Waiting for close lock at 1731569112806Running coprocessor pre-close hooks at 1731569112806Disabling compacts and flushes for region at 1731569112806Disabling writes for close at 1731569112806Obtaining lock to block concurrent updates at 1731569112807 (+1 ms)Preparing flush snapshotting stores in eb8aa6a5cb82db71eb9aa5187f031ed4 at 1731569112807Finished memstore snapshotting TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731569112807Flushing stores of TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4. at 1731569112808 (+1 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/A: creating writer at 1731569112808Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/A: appending metadata at 1731569112812 (+4 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/A: closing flushed file at 1731569112812Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/B: creating writer at 1731569113221 (+409 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/B: appending metadata at 1731569113229 (+8 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/B: closing flushed file at 1731569113229Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/C: creating writer at 1731569113641 (+412 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/C: appending metadata at 1731569113648 (+7 ms)Flushing eb8aa6a5cb82db71eb9aa5187f031ed4/C: closing flushed file at 1731569113649 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8a4797: reopening flushed file at 1731569114067 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1609246f: reopening flushed file at 1731569114077 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74a15860: reopening flushed file at 1731569114084 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for eb8aa6a5cb82db71eb9aa5187f031ed4 in 1286ms, sequenceid=627, compaction requested=true at 1731569114092 (+8 ms)Writing region close event to WAL at 1731569114296 (+204 ms)Running coprocessor post-close hooks at 1731569114300 (+4 ms)Closed at 1731569114300 2024-11-14T07:25:14,303 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:14,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=eb8aa6a5cb82db71eb9aa5187f031ed4, regionState=CLOSED 2024-11-14T07:25:14,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:14,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T07:25:14,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure eb8aa6a5cb82db71eb9aa5187f031ed4, server=c7aa6e2790b3,46235,1731569085523 in 1.6610 sec 2024-11-14T07:25:14,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T07:25:14,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eb8aa6a5cb82db71eb9aa5187f031ed4, UNASSIGN in 1.6690 sec 2024-11-14T07:25:14,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T07:25:14,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6760 sec 2024-11-14T07:25:14,313 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569114313"}]},"ts":"1731569114313"} 2024-11-14T07:25:14,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:25:14,315 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:25:14,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6890 sec 2024-11-14T07:25:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T07:25:14,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:14,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:25:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,784 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-14T07:25:14,786 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,791 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:14,797 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/recovered.edits] 2024-11-14T07:25:14,802 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/3cc0f02d061d47b7b3f0da89043bfc93 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/3cc0f02d061d47b7b3f0da89043bfc93 2024-11-14T07:25:14,804 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7ec000497c8948acb254c35ea59fd25a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/7ec000497c8948acb254c35ea59fd25a 2024-11-14T07:25:14,807 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/974706908f65434ca5bd641d7e84233e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/A/974706908f65434ca5bd641d7e84233e 2024-11-14T07:25:14,810 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/d862c9cf06fc409ab8f4d2d1a15a2c71 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/d862c9cf06fc409ab8f4d2d1a15a2c71 2024-11-14T07:25:14,811 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/df64867d4a7f444582517bd217516bfd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/df64867d4a7f444582517bd217516bfd 2024-11-14T07:25:14,813 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f0002035838746d780b3fbd2467d9764 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/B/f0002035838746d780b3fbd2467d9764 2024-11-14T07:25:14,816 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/19ec5603437540419aa48dc2be0fc4f6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/19ec5603437540419aa48dc2be0fc4f6 2024-11-14T07:25:14,818 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7f2f402c82e3401883af9f425112745b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/7f2f402c82e3401883af9f425112745b 2024-11-14T07:25:14,819 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/ba171db053634e5d97899332d36c3b3c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/C/ba171db053634e5d97899332d36c3b3c 2024-11-14T07:25:14,822 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/recovered.edits/630.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4/recovered.edits/630.seqid 2024-11-14T07:25:14,823 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/eb8aa6a5cb82db71eb9aa5187f031ed4 2024-11-14T07:25:14,823 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:25:14,828 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T07:25:14,838 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:25:14,863 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:25:14,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:25:14,864 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569114864"}]},"ts":"9223372036854775807"} 2024-11-14T07:25:14,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:25:14,869 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eb8aa6a5cb82db71eb9aa5187f031ed4, NAME => 'TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:25:14,869 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:25:14,869 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569114869"}]},"ts":"9223372036854775807"} 2024-11-14T07:25:14,873 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:25:14,874 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-14T07:25:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-14T07:25:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:25:14,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:14,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c7aa6e2790b3:46235-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1194628676_22 at /127.0.0.1:57950 [Waiting for operation #70] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_230653519_22 at /127.0.0.1:38876 [Waiting for operation #384] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_230653519_22 at /127.0.0.1:44316 [Waiting for operation #400] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 320) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6148 (was 7126) 2024-11-14T07:25:14,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=6148 2024-11-14T07:25:14,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:25:14,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:25:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:14,916 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:25:14,916 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:14,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-14T07:25:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:14,917 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:25:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741971_1147 (size=959) 2024-11-14T07:25:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:15,107 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T07:25:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:15,329 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:25:15,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741972_1148 (size=53) 2024-11-14T07:25:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c7069097a7b68031cda66e228521c56e, disabling compactions & flushes 2024-11-14T07:25:15,744 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. after waiting 0 ms 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:15,744 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:15,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c7069097a7b68031cda66e228521c56e: Waiting for close lock at 1731569115744Disabling compacts and flushes for region at 1731569115744Disabling writes for close at 1731569115744Writing region close event to WAL at 1731569115744Closed at 1731569115744 2024-11-14T07:25:15,747 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:25:15,748 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569115747"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569115747"}]},"ts":"1731569115747"} 2024-11-14T07:25:15,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:25:15,753 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:25:15,754 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569115753"}]},"ts":"1731569115753"} 2024-11-14T07:25:15,755 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:25:15,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, ASSIGN}] 2024-11-14T07:25:15,757 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, ASSIGN 2024-11-14T07:25:15,758 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:25:15,909 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:15,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, ASSIGN because future has completed 2024-11-14T07:25:15,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:16,071 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:16,071 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:25:16,072 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,072 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:25:16,072 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,072 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,073 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,075 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName A 2024-11-14T07:25:16,075 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:16,076 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:16,076 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,077 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName B 2024-11-14T07:25:16,077 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:16,078 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:16,078 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,079 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName C 2024-11-14T07:25:16,079 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:16,079 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:16,079 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,080 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,080 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,081 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,081 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,082 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:25:16,083 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,085 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:25:16,085 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c7069097a7b68031cda66e228521c56e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67525614, jitterRate=0.006210058927536011}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:25:16,085 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:16,086 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c7069097a7b68031cda66e228521c56e: Running coprocessor pre-open hook at 1731569116072Writing region info on filesystem at 1731569116072Initializing all the Stores at 1731569116073 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569116073Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569116073Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569116073Cleaning up temporary data from old regions at 1731569116081 (+8 ms)Running coprocessor post-open hooks at 1731569116085 (+4 ms)Region opened successfully at 1731569116086 (+1 ms) 2024-11-14T07:25:16,087 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., pid=36, masterSystemTime=1731569116068 2024-11-14T07:25:16,089 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:16,089 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:16,089 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:16,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:16,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T07:25:16,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 in 177 msec 2024-11-14T07:25:16,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T07:25:16,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, ASSIGN in 338 msec 2024-11-14T07:25:16,096 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:25:16,097 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569116097"}]},"ts":"1731569116097"} 2024-11-14T07:25:16,098 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:25:16,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:25:16,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-14T07:25:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T07:25:17,076 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:17,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:17,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:17,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:17,084 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e4063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:17,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@610ba11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:17,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:17,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:17,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:25:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:25:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@412d21f3 2024-11-14T07:25:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:25:17,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:25:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:25:17,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:25:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:17,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741973_1149 (size=995) 2024-11-14T07:25:17,536 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T07:25:17,536 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T07:25:17,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:25:17,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE}] 2024-11-14T07:25:17,551 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE 2024-11-14T07:25:17,551 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:17,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE because future has completed 2024-11-14T07:25:17,553 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:25:17,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:17,707 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:17,708 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:25:17,708 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing c7069097a7b68031cda66e228521c56e, disabling compactions & flushes 2024-11-14T07:25:17,708 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:17,708 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:17,708 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. after waiting 0 ms 2024-11-14T07:25:17,708 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:17,717 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:25:17,718 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:17,718 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for c7069097a7b68031cda66e228521c56e: Waiting for close lock at 1731569117708Running coprocessor pre-close hooks at 1731569117708Disabling compacts and flushes for region at 1731569117708Disabling writes for close at 1731569117708Writing region close event to WAL at 1731569117709 (+1 ms)Running coprocessor post-close hooks at 1731569117718 (+9 ms)Closed at 1731569117718 2024-11-14T07:25:17,718 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: c7069097a7b68031cda66e228521c56e to self. 2024-11-14T07:25:17,721 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:17,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=CLOSED 2024-11-14T07:25:17,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:17,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-14T07:25:17,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 in 171 msec 2024-11-14T07:25:17,728 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=true 2024-11-14T07:25:17,879 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:17,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE because future has completed 2024-11-14T07:25:17,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:18,040 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,040 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:25:18,040 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,040 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:25:18,040 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,040 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,042 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,046 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName A 2024-11-14T07:25:18,048 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:18,049 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:18,049 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,051 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName B 2024-11-14T07:25:18,051 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:18,051 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:18,052 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,053 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7069097a7b68031cda66e228521c56e columnFamilyName C 2024-11-14T07:25:18,053 DEBUG [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:18,053 INFO [StoreOpener-c7069097a7b68031cda66e228521c56e-1 {}] regionserver.HStore(327): Store=c7069097a7b68031cda66e228521c56e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:18,053 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,054 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,055 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,056 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,056 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,056 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:25:18,058 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,059 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c7069097a7b68031cda66e228521c56e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63484296, jitterRate=-0.05401027202606201}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:25:18,059 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,059 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c7069097a7b68031cda66e228521c56e: Running coprocessor pre-open hook at 1731569118040Writing region info on filesystem at 1731569118040Initializing all the Stores at 1731569118041 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569118041Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569118041Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569118041Cleaning up temporary data from old regions at 1731569118056 (+15 ms)Running coprocessor post-open hooks at 1731569118059 (+3 ms)Region opened successfully at 1731569118059 2024-11-14T07:25:18,060 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., pid=41, masterSystemTime=1731569118037 2024-11-14T07:25:18,062 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,062 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,062 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:18,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-14T07:25:18,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 in 182 msec 2024-11-14T07:25:18,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T07:25:18,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, REOPEN/MOVE in 517 msec 2024-11-14T07:25:18,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-14T07:25:18,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-14T07:25:18,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 969 msec 2024-11-14T07:25:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-14T07:25:18,075 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:18,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,077 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a04ca1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,078 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab3e094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,080 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3368167d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,082 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c80a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,084 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f187713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,086 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b0574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,088 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@768c6b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,089 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7417e10d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,091 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588eedce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,092 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd9ff4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,094 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de68372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cb5082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f42cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228997ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eeff5bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57bfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:18,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:18,105 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469ba3b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:18,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e7d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:18,108 DEBUG [Thread-733 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 DEBUG [Thread-735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 DEBUG [Thread-744 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 DEBUG [Thread-738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 DEBUG [Thread-727 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 DEBUG [Thread-740 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:18,109 DEBUG [Thread-729 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,109 DEBUG [Thread-731 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,109 DEBUG [Thread-742 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,110 DEBUG [Thread-738 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,110 DEBUG [Thread-738 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,111 DEBUG [Thread-744 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,111 DEBUG [Thread-744 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T07:25:18,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:18,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:18,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:18,114 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:18,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:25:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:18,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:18,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb89a837402243ada65cf420a5e39183_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569118119/Put/seqid=0 2024-11-14T07:25:18,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741974_1150 (size=12154) 2024-11-14T07:25:18,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569178181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569178183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569178190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569178189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569178202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:18,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569178316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569178317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569178317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569178317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569178317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:18,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:18,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569178534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569178534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569178535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569178535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569178535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:18,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:18,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:18,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb89a837402243ada65cf420a5e39183_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb89a837402243ada65cf420a5e39183_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:18,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/84e81dc12e754a0bb99aef107ef1e87f, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:18,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/84e81dc12e754a0bb99aef107ef1e87f is 175, key is test_row_0/A:col10/1731569118119/Put/seqid=0 2024-11-14T07:25:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741975_1151 (size=30955) 2024-11-14T07:25:18,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:18,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569178846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569178846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569178846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569178846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569178846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:18,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:18,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:18,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/84e81dc12e754a0bb99aef107ef1e87f 2024-11-14T07:25:19,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ca899333058a4b33a2aa0a42ff407324 is 50, key is test_row_0/B:col10/1731569118119/Put/seqid=0 2024-11-14T07:25:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741976_1152 (size=12001) 2024-11-14T07:25:19,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ca899333058a4b33a2aa0a42ff407324 2024-11-14T07:25:19,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1454cba5b9a8416ba85b45803a66b4ea is 50, key is test_row_0/C:col10/1731569118119/Put/seqid=0 2024-11-14T07:25:19,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741977_1153 (size=12001) 2024-11-14T07:25:19,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1454cba5b9a8416ba85b45803a66b4ea 2024-11-14T07:25:19,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/84e81dc12e754a0bb99aef107ef1e87f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f 2024-11-14T07:25:19,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f, entries=150, sequenceid=15, filesize=30.2 K 2024-11-14T07:25:19,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ca899333058a4b33a2aa0a42ff407324 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324 2024-11-14T07:25:19,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:25:19,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1454cba5b9a8416ba85b45803a66b4ea as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea 2024-11-14T07:25:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:19,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:25:19,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c7069097a7b68031cda66e228521c56e in 1135ms, sequenceid=15, compaction requested=false 2024-11-14T07:25:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:19,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T07:25:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:25:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:19,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb46496dc957470c96ef9f0a85b37b6d_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569118188/Put/seqid=0 2024-11-14T07:25:19,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569179375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569179379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569179382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569179388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569179389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741978_1154 (size=12154) 2024-11-14T07:25:19,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:19,410 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb46496dc957470c96ef9f0a85b37b6d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb46496dc957470c96ef9f0a85b37b6d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:19,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/0b3579cbf66b4ad3ac740f7f010296de, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/0b3579cbf66b4ad3ac740f7f010296de is 175, key is test_row_0/A:col10/1731569118188/Put/seqid=0 2024-11-14T07:25:19,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741979_1155 (size=30955) 2024-11-14T07:25:19,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/0b3579cbf66b4ad3ac740f7f010296de 2024-11-14T07:25:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf034d5b59b44e2f982cae3f8bdfc857 is 50, key is test_row_0/B:col10/1731569118188/Put/seqid=0 2024-11-14T07:25:19,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569179495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569179496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569179497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741980_1156 (size=12001) 2024-11-14T07:25:19,503 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf034d5b59b44e2f982cae3f8bdfc857 2024-11-14T07:25:19,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569179505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569179506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4271b4ff03ee463590718012702089dd is 50, key is test_row_0/C:col10/1731569118188/Put/seqid=0 2024-11-14T07:25:19,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741981_1157 (size=12001) 2024-11-14T07:25:19,549 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4271b4ff03ee463590718012702089dd 2024-11-14T07:25:19,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/0b3579cbf66b4ad3ac740f7f010296de as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de 2024-11-14T07:25:19,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de, entries=150, sequenceid=40, filesize=30.2 K 2024-11-14T07:25:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf034d5b59b44e2f982cae3f8bdfc857 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857 2024-11-14T07:25:19,594 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T07:25:19,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4271b4ff03ee463590718012702089dd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd 2024-11-14T07:25:19,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T07:25:19,605 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c7069097a7b68031cda66e228521c56e in 253ms, sequenceid=40, compaction requested=false 2024-11-14T07:25:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T07:25:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T07:25:19,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T07:25:19,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-11-14T07:25:19,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5010 sec 2024-11-14T07:25:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:19,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:19,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:25:19,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a7399b3c54ec4902936262c35429191f_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:19,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569179739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569179742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741982_1158 (size=14594) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569179742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569179744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569179744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:19,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a7399b3c54ec4902936262c35429191f_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a7399b3c54ec4902936262c35429191f_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:19,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/461527fc4a7c4fb8b5722295782d62b6, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:19,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/461527fc4a7c4fb8b5722295782d62b6 is 175, key is test_row_0/A:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:19,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741983_1159 (size=39549) 2024-11-14T07:25:19,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/461527fc4a7c4fb8b5722295782d62b6 2024-11-14T07:25:19,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5266ecb7091f46169d0018f7f967fec6 is 50, key is test_row_0/B:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569179854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569179854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569179855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569179855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569179855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:19,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:19,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741984_1160 (size=12001) 2024-11-14T07:25:19,879 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:25:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569180065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569180065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569180065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569180066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569180066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T07:25:20,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:20,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T07:25:20,267 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:20,268 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:20,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5266ecb7091f46169d0018f7f967fec6 2024-11-14T07:25:20,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:20,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1ad88b4f1f9b497eae6efe8fc6b12166 is 50, key is test_row_0/C:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:20,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741985_1161 (size=12001) 2024-11-14T07:25:20,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1ad88b4f1f9b497eae6efe8fc6b12166 2024-11-14T07:25:20,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/461527fc4a7c4fb8b5722295782d62b6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6 2024-11-14T07:25:20,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6, entries=200, sequenceid=53, filesize=38.6 K 2024-11-14T07:25:20,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5266ecb7091f46169d0018f7f967fec6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6 2024-11-14T07:25:20,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T07:25:20,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/1ad88b4f1f9b497eae6efe8fc6b12166 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166 2024-11-14T07:25:20,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T07:25:20,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c7069097a7b68031cda66e228521c56e in 612ms, sequenceid=53, compaction requested=true 2024-11-14T07:25:20,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:20,319 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:20,319 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:20,325 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:20,326 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:20,326 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,326 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=99.1 K 2024-11-14T07:25:20,326 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,326 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6] 2024-11-14T07:25:20,328 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:20,328 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e81dc12e754a0bb99aef107ef1e87f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569118119 2024-11-14T07:25:20,328 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:20,328 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,328 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.2 K 2024-11-14T07:25:20,329 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3579cbf66b4ad3ac740f7f010296de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569118181 2024-11-14T07:25:20,329 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ca899333058a4b33a2aa0a42ff407324, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569118119 2024-11-14T07:25:20,329 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461527fc4a7c4fb8b5722295782d62b6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119384 2024-11-14T07:25:20,330 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting cf034d5b59b44e2f982cae3f8bdfc857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569118181 2024-11-14T07:25:20,331 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5266ecb7091f46169d0018f7f967fec6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119385 2024-11-14T07:25:20,341 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:20,343 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:20,343 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d27de005e95f4a34beeef220bb18f95a is 50, key is test_row_0/B:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:20,353 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e6bfa75bae414182a15c1bdd15aeec63_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:20,365 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e6bfa75bae414182a15c1bdd15aeec63_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:20,365 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e6bfa75bae414182a15c1bdd15aeec63_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:20,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:20,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:25:20,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741986_1162 (size=12104) 2024-11-14T07:25:20,392 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d27de005e95f4a34beeef220bb18f95a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d27de005e95f4a34beeef220bb18f95a 2024-11-14T07:25:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569180393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,401 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into d27de005e95f4a34beeef220bb18f95a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:20,401 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:20,401 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569120319; duration=0sec 2024-11-14T07:25:20,401 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:20,401 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:20,401 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569180393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569180395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569180396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,405 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:20,405 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:20,405 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,405 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.2 K 2024-11-14T07:25:20,405 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1454cba5b9a8416ba85b45803a66b4ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569118119 2024-11-14T07:25:20,406 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4271b4ff03ee463590718012702089dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569118181 2024-11-14T07:25:20,406 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ad88b4f1f9b497eae6efe8fc6b12166, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119385 2024-11-14T07:25:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569180401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741987_1163 (size=4469) 2024-11-14T07:25:20,431 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#144 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:20,431 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/307b385a13924b52b8ba82ceff4d005a is 175, key is test_row_0/A:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:20,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148313f26e720c46da98b7264d99cc9d17_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569120377/Put/seqid=0 2024-11-14T07:25:20,463 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:20,463 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/35df91d6f7fa42ff83a9c8c0adef11fd is 50, key is test_row_0/C:col10/1731569119705/Put/seqid=0 2024-11-14T07:25:20,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741988_1164 (size=31058) 2024-11-14T07:25:20,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741989_1165 (size=19474) 2024-11-14T07:25:20,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741990_1166 (size=12104) 2024-11-14T07:25:20,498 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/35df91d6f7fa42ff83a9c8c0adef11fd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/35df91d6f7fa42ff83a9c8c0adef11fd 2024-11-14T07:25:20,505 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 35df91d6f7fa42ff83a9c8c0adef11fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:20,505 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:20,505 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569120319; duration=0sec 2024-11-14T07:25:20,505 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:20,505 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569180504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569180504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569180504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569180504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569180515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569180715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569180715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569180715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569180715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569180725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:20,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:20,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:20,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:20,904 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148313f26e720c46da98b7264d99cc9d17_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148313f26e720c46da98b7264d99cc9d17_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:20,905 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/307b385a13924b52b8ba82ceff4d005a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a 2024-11-14T07:25:20,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/651c0b2605794fb695dc0a686b91e94c, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:20,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/651c0b2605794fb695dc0a686b91e94c is 175, key is test_row_0/A:col10/1731569120377/Put/seqid=0 2024-11-14T07:25:20,919 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 307b385a13924b52b8ba82ceff4d005a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:20,919 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:20,919 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569120319; duration=0sec 2024-11-14T07:25:20,919 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:20,920 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:20,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741991_1167 (size=56733) 2024-11-14T07:25:20,929 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/651c0b2605794fb695dc0a686b91e94c 2024-11-14T07:25:20,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/23bc8e341b0443ffaddf4a63650e69a3 is 50, key is test_row_0/B:col10/1731569120377/Put/seqid=0 2024-11-14T07:25:21,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741992_1168 (size=12001) 2024-11-14T07:25:21,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/23bc8e341b0443ffaddf4a63650e69a3 2024-11-14T07:25:21,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f21547976e134c79a788e822aff8bab3 is 50, key is test_row_0/C:col10/1731569120377/Put/seqid=0 2024-11-14T07:25:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569181025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569181025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569181025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569181034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569181034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741993_1169 (size=12001) 2024-11-14T07:25:21,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:21,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f21547976e134c79a788e822aff8bab3 2024-11-14T07:25:21,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/651c0b2605794fb695dc0a686b91e94c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c 2024-11-14T07:25:21,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c, entries=300, sequenceid=79, filesize=55.4 K 2024-11-14T07:25:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/23bc8e341b0443ffaddf4a63650e69a3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3 2024-11-14T07:25:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T07:25:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f21547976e134c79a788e822aff8bab3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3 2024-11-14T07:25:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T07:25:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c7069097a7b68031cda66e228521c56e in 1126ms, sequenceid=79, compaction requested=false 2024-11-14T07:25:21,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T07:25:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T07:25:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ebb78747a74a4a2fb78a9729c5dc08b2_c7069097a7b68031cda66e228521c56e is 50, key is test_row_1/A:col10/1731569120393/Put/seqid=0 2024-11-14T07:25:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741994_1170 (size=9714) 2024-11-14T07:25:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ebb78747a74a4a2fb78a9729c5dc08b2_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ebb78747a74a4a2fb78a9729c5dc08b2_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/779a740a0b4344c2bab12fcd4713bcea, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/779a740a0b4344c2bab12fcd4713bcea is 175, key is test_row_1/A:col10/1731569120393/Put/seqid=0 2024-11-14T07:25:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741995_1171 (size=22361) 2024-11-14T07:25:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/779a740a0b4344c2bab12fcd4713bcea 2024-11-14T07:25:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a13c5e4c965b4ddbb2921623d0fddd3d is 50, key is test_row_1/B:col10/1731569120393/Put/seqid=0 2024-11-14T07:25:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741996_1172 (size=9657) 2024-11-14T07:25:21,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a13c5e4c965b4ddbb2921623d0fddd3d 2024-11-14T07:25:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/2f0669f27149483887970d99e01974a2 is 50, key is test_row_1/C:col10/1731569120393/Put/seqid=0 2024-11-14T07:25:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741997_1173 (size=9657) 2024-11-14T07:25:21,614 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/2f0669f27149483887970d99e01974a2 2024-11-14T07:25:21,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/779a740a0b4344c2bab12fcd4713bcea as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea 2024-11-14T07:25:21,626 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea, entries=100, sequenceid=92, filesize=21.8 K 2024-11-14T07:25:21,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a13c5e4c965b4ddbb2921623d0fddd3d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d 2024-11-14T07:25:21,635 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d, entries=100, sequenceid=92, filesize=9.4 K 2024-11-14T07:25:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/2f0669f27149483887970d99e01974a2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2 2024-11-14T07:25:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2, entries=100, sequenceid=92, filesize=9.4 K 2024-11-14T07:25:21,644 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=46.96 KB/48090 for c7069097a7b68031cda66e228521c56e in 133ms, sequenceid=92, compaction requested=true 2024-11-14T07:25:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T07:25:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T07:25:21,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T07:25:21,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3780 sec 2024-11-14T07:25:21,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3850 sec 2024-11-14T07:25:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,655 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:21,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:21,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114877e30a4e50c4605810da9b5b61e10f3_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741998_1174 (size=17034) 2024-11-14T07:25:21,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114877e30a4e50c4605810da9b5b61e10f3_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114877e30a4e50c4605810da9b5b61e10f3_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/654a51a889ed4dcd85122195b96d1989, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/654a51a889ed4dcd85122195b96d1989 is 175, key is test_row_0/A:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741999_1175 (size=48139) 2024-11-14T07:25:21,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/654a51a889ed4dcd85122195b96d1989 2024-11-14T07:25:21,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/883ed4e3a0554b11b7d2b88c00ad0adf is 50, key is test_row_0/B:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742000_1176 (size=12001) 2024-11-14T07:25:21,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/883ed4e3a0554b11b7d2b88c00ad0adf 2024-11-14T07:25:21,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/de9dcc6182424bcfba53b9c790fdf0aa is 50, key is test_row_0/C:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742001_1177 (size=12001) 2024-11-14T07:25:21,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/de9dcc6182424bcfba53b9c790fdf0aa 2024-11-14T07:25:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/654a51a889ed4dcd85122195b96d1989 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989 2024-11-14T07:25:21,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989, entries=250, sequenceid=103, filesize=47.0 K 2024-11-14T07:25:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/883ed4e3a0554b11b7d2b88c00ad0adf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf 2024-11-14T07:25:21,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf, entries=150, sequenceid=103, filesize=11.7 K 2024-11-14T07:25:21,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/de9dcc6182424bcfba53b9c790fdf0aa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa 2024-11-14T07:25:21,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa, entries=150, sequenceid=103, filesize=11.7 K 2024-11-14T07:25:21,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=93.93 KB/96180 for c7069097a7b68031cda66e228521c56e in 114ms, sequenceid=103, compaction requested=true 2024-11-14T07:25:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:21,769 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:21,769 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:21,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:21,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:21,772 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158291 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:21,772 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:21,772 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,772 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=154.6 K 2024-11-14T07:25:21,772 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,772 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989] 2024-11-14T07:25:21,775 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:21,775 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 307b385a13924b52b8ba82ceff4d005a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119385 2024-11-14T07:25:21,776 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:21,776 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,776 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d27de005e95f4a34beeef220bb18f95a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=44.7 K 2024-11-14T07:25:21,776 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 651c0b2605794fb695dc0a686b91e94c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731569119729 2024-11-14T07:25:21,776 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d27de005e95f4a34beeef220bb18f95a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119385 2024-11-14T07:25:21,777 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779a740a0b4344c2bab12fcd4713bcea, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569120393 2024-11-14T07:25:21,777 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bc8e341b0443ffaddf4a63650e69a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731569119729 2024-11-14T07:25:21,777 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654a51a889ed4dcd85122195b96d1989, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:21,777 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a13c5e4c965b4ddbb2921623d0fddd3d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569120393 2024-11-14T07:25:21,779 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 883ed4e3a0554b11b7d2b88c00ad0adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:21,797 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:21,798 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/392f175a8044494880bdfa099931a3ba is 50, key is test_row_0/B:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111490812aed87bc45f99a3aa08338391d4d_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569121770/Put/seqid=0 2024-11-14T07:25:21,814 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114fb2d8b98b25b426a99df7829383da834_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,825 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114fb2d8b98b25b426a99df7829383da834_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,825 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fb2d8b98b25b426a99df7829383da834_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569181817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569181825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569181828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569181829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742002_1178 (size=21918) 2024-11-14T07:25:21,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569181835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:21,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111490812aed87bc45f99a3aa08338391d4d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111490812aed87bc45f99a3aa08338391d4d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:21,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bb289b9688484c3bbca02940544948bb, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:21,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bb289b9688484c3bbca02940544948bb is 175, key is test_row_0/A:col10/1731569121770/Put/seqid=0 2024-11-14T07:25:21,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742003_1179 (size=12241) 2024-11-14T07:25:21,861 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/392f175a8044494880bdfa099931a3ba as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/392f175a8044494880bdfa099931a3ba 2024-11-14T07:25:21,868 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into 392f175a8044494880bdfa099931a3ba(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:21,868 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:21,868 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=12, startTime=1731569121769; duration=0sec 2024-11-14T07:25:21,868 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:21,868 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:21,869 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:21,870 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:21,870 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:21,870 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:21,870 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/35df91d6f7fa42ff83a9c8c0adef11fd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=44.7 K 2024-11-14T07:25:21,872 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 35df91d6f7fa42ff83a9c8c0adef11fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731569119385 2024-11-14T07:25:21,872 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f21547976e134c79a788e822aff8bab3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731569119729 2024-11-14T07:25:21,873 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f0669f27149483887970d99e01974a2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569120393 2024-11-14T07:25:21,873 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting de9dcc6182424bcfba53b9c790fdf0aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:21,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742004_1180 (size=4469) 2024-11-14T07:25:21,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742005_1181 (size=65323) 2024-11-14T07:25:21,903 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bb289b9688484c3bbca02940544948bb 2024-11-14T07:25:21,914 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:21,915 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/9a371ccc2f7b4637ae11aacd99841baa is 50, key is test_row_0/C:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:21,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/f79b55b34bf24dd88d323323abc1dbe0 is 50, key is test_row_0/B:col10/1731569121770/Put/seqid=0 2024-11-14T07:25:21,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742006_1182 (size=12001) 2024-11-14T07:25:21,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569181935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569181935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569181945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569181945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742007_1183 (size=12241) 2024-11-14T07:25:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569181945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:21,957 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/9a371ccc2f7b4637ae11aacd99841baa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9a371ccc2f7b4637ae11aacd99841baa 2024-11-14T07:25:21,966 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 9a371ccc2f7b4637ae11aacd99841baa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:21,966 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:21,966 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=12, startTime=1731569121770; duration=0sec 2024-11-14T07:25:21,966 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:21,966 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569182145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569182145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569182154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569182154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569182154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,292 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#157 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:22,292 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/094bf1cdd8cc47fa966e925b690a8829 is 175, key is test_row_0/A:col10/1731569121648/Put/seqid=0 2024-11-14T07:25:22,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742008_1184 (size=31195) 2024-11-14T07:25:22,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/f79b55b34bf24dd88d323323abc1dbe0 2024-11-14T07:25:22,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/039d0c7fa0874d5aa440795bd3bb9894 is 50, key is test_row_0/C:col10/1731569121770/Put/seqid=0 2024-11-14T07:25:22,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742009_1185 (size=12001) 2024-11-14T07:25:22,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/039d0c7fa0874d5aa440795bd3bb9894 2024-11-14T07:25:22,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bb289b9688484c3bbca02940544948bb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb 2024-11-14T07:25:22,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb, entries=350, sequenceid=123, filesize=63.8 K 2024-11-14T07:25:22,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/f79b55b34bf24dd88d323323abc1dbe0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0 2024-11-14T07:25:22,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T07:25:22,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/039d0c7fa0874d5aa440795bd3bb9894 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894 2024-11-14T07:25:22,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T07:25:22,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for c7069097a7b68031cda66e228521c56e in 625ms, sequenceid=123, compaction requested=false 2024-11-14T07:25:22,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T07:25:22,415 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:22,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T07:25:22,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:22,420 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:22,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:22,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:22,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e50f5d68474ee9bb59d9620ab64f9d_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:22,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742010_1186 (size=14744) 2024-11-14T07:25:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569182508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569182508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569182510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569182510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569182511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:22,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569182615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569182626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,704 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/094bf1cdd8cc47fa966e925b690a8829 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829 2024-11-14T07:25:22,712 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 094bf1cdd8cc47fa966e925b690a8829(size=30.5 K), total size for store is 94.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:22,712 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:22,712 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=12, startTime=1731569121769; duration=0sec 2024-11-14T07:25:22,712 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:22,712 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:22,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569182824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569182844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569182844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569182845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569182846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:22,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:22,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:22,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:22,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e50f5d68474ee9bb59d9620ab64f9d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e50f5d68474ee9bb59d9620ab64f9d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:22,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8bfc758b873d418682743e870a40b861, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:22,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8bfc758b873d418682743e870a40b861 is 175, key is test_row_0/A:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:22,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742011_1187 (size=39699) 2024-11-14T07:25:23,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569183134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569183154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569183155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569183155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569183155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8bfc758b873d418682743e870a40b861 2024-11-14T07:25:23,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/04195f54032b44bba18d7969363686af is 50, key is test_row_0/B:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:23,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742012_1188 (size=12151) 2024-11-14T07:25:23,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/04195f54032b44bba18d7969363686af 2024-11-14T07:25:23,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/5e9ebf22722244f28068278f7282f196 is 50, key is test_row_0/C:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742013_1189 (size=12151) 2024-11-14T07:25:23,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/5e9ebf22722244f28068278f7282f196 2024-11-14T07:25:23,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8bfc758b873d418682743e870a40b861 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861 2024-11-14T07:25:23,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:23,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861, entries=200, sequenceid=143, filesize=38.8 K 2024-11-14T07:25:23,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/04195f54032b44bba18d7969363686af as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af 2024-11-14T07:25:23,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T07:25:23,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/5e9ebf22722244f28068278f7282f196 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196 2024-11-14T07:25:23,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T07:25:23,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c7069097a7b68031cda66e228521c56e in 912ms, sequenceid=143, compaction requested=true 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:23,369 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:23,369 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:23,372 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:23,372 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:23,372 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,372 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=133.0 K 2024-11-14T07:25:23,372 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,372 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861] 2024-11-14T07:25:23,373 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094bf1cdd8cc47fa966e925b690a8829, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:23,375 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:23,375 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:23,375 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,375 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/392f175a8044494880bdfa099931a3ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.5 K 2024-11-14T07:25:23,375 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb289b9688484c3bbca02940544948bb, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731569121684 2024-11-14T07:25:23,376 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 392f175a8044494880bdfa099931a3ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:23,376 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfc758b873d418682743e870a40b861, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121780 2024-11-14T07:25:23,377 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f79b55b34bf24dd88d323323abc1dbe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731569121765 2024-11-14T07:25:23,377 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 04195f54032b44bba18d7969363686af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121828 2024-11-14T07:25:23,400 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:23,400 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a0fa28d7d8594239be997acb479b1368 is 50, key is test_row_0/B:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:23,405 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:23,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145820f213683442a596ff4ba090a8dc1b_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:23,416 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145820f213683442a596ff4ba090a8dc1b_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:23,416 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145820f213683442a596ff4ba090a8dc1b_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742014_1190 (size=12493) 2024-11-14T07:25:23,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742015_1191 (size=4469) 2024-11-14T07:25:23,426 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#165 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:23,427 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8105df8523a34c568b29035b69f19288 is 175, key is test_row_0/A:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:23,427 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a0fa28d7d8594239be997acb479b1368 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a0fa28d7d8594239be997acb479b1368 2024-11-14T07:25:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742016_1192 (size=31447) 2024-11-14T07:25:23,434 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into a0fa28d7d8594239be997acb479b1368(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:23,434 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:23,434 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569123369; duration=0sec 2024-11-14T07:25:23,434 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:23,434 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:23,434 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:23,435 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:23,435 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:23,435 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,435 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9a371ccc2f7b4637ae11aacd99841baa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.5 K 2024-11-14T07:25:23,435 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a371ccc2f7b4637ae11aacd99841baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731569121604 2024-11-14T07:25:23,436 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 039d0c7fa0874d5aa440795bd3bb9894, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731569121765 2024-11-14T07:25:23,436 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9ebf22722244f28068278f7282f196, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121828 2024-11-14T07:25:23,443 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/8105df8523a34c568b29035b69f19288 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288 2024-11-14T07:25:23,449 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:23,449 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 8105df8523a34c568b29035b69f19288(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:23,449 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:23,449 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569123369; duration=0sec 2024-11-14T07:25:23,449 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:23,449 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:23,449 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6c8166abe60b491aa524c7df9c7e5cae is 50, key is test_row_0/C:col10/1731569121828/Put/seqid=0 2024-11-14T07:25:23,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742017_1193 (size=12493) 2024-11-14T07:25:23,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T07:25:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:23,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:23,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444ac7e9540b84e6b88ca985c02c4a0f7_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569122508/Put/seqid=0 2024-11-14T07:25:23,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742018_1194 (size=12304) 2024-11-14T07:25:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:23,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444ac7e9540b84e6b88ca985c02c4a0f7_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444ac7e9540b84e6b88ca985c02c4a0f7_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:23,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/e4337256b6ee4ec88912a2fe786d6f44, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/e4337256b6ee4ec88912a2fe786d6f44 is 175, key is test_row_0/A:col10/1731569122508/Put/seqid=0 2024-11-14T07:25:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:23,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742019_1195 (size=31105) 2024-11-14T07:25:23,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/e4337256b6ee4ec88912a2fe786d6f44 2024-11-14T07:25:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/1bf7e345369a46ba8433d1e601ddbdaf is 50, key is test_row_0/B:col10/1731569122508/Put/seqid=0 2024-11-14T07:25:23,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742020_1196 (size=12151) 2024-11-14T07:25:23,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:23,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569183673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569183674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569183674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569183676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569183681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569183785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569183785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569183785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569183795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569183795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,859 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6c8166abe60b491aa524c7df9c7e5cae as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6c8166abe60b491aa524c7df9c7e5cae 2024-11-14T07:25:23,867 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 6c8166abe60b491aa524c7df9c7e5cae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:23,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:23,867 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569123369; duration=0sec 2024-11-14T07:25:23,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:23,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:23,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/1bf7e345369a46ba8433d1e601ddbdaf 2024-11-14T07:25:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569183994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569183995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569183995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6cd01df466bc48d99710ee14e67d63fb is 50, key is test_row_0/C:col10/1731569122508/Put/seqid=0 2024-11-14T07:25:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569184004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569184004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742021_1197 (size=12151) 2024-11-14T07:25:24,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6cd01df466bc48d99710ee14e67d63fb 2024-11-14T07:25:24,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/e4337256b6ee4ec88912a2fe786d6f44 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44 2024-11-14T07:25:24,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44, entries=150, sequenceid=164, filesize=30.4 K 2024-11-14T07:25:24,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/1bf7e345369a46ba8433d1e601ddbdaf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf 2024-11-14T07:25:24,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf, entries=150, sequenceid=164, filesize=11.9 K 2024-11-14T07:25:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6cd01df466bc48d99710ee14e67d63fb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb 2024-11-14T07:25:24,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb, entries=150, sequenceid=164, filesize=11.9 K 2024-11-14T07:25:24,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c7069097a7b68031cda66e228521c56e in 559ms, sequenceid=164, compaction requested=false 2024-11-14T07:25:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T07:25:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T07:25:24,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T07:25:24,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-14T07:25:24,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6490 sec 2024-11-14T07:25:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:24,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:24,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T07:25:24,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f275754e2d7a4ceab52e9b03e8030fd1_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569184324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569184325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569184326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569184325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569184326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742022_1198 (size=19774) 2024-11-14T07:25:24,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:24,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f275754e2d7a4ceab52e9b03e8030fd1_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f275754e2d7a4ceab52e9b03e8030fd1_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:24,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3adbe23e9c66458bb82fdb20c002a015, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:24,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3adbe23e9c66458bb82fdb20c002a015 is 175, key is test_row_0/A:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:24,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742023_1199 (size=57033) 2024-11-14T07:25:24,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3adbe23e9c66458bb82fdb20c002a015 2024-11-14T07:25:24,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ad0d190b25b94785be7f2bb58c709b28 is 50, key is test_row_0/B:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:24,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742024_1200 (size=12151) 2024-11-14T07:25:24,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569184434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569184435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569184434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569184435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569184435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T07:25:24,564 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:24,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T07:25:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:24,566 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:24,567 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:24,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569184645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569184645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569184645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569184645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569184645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:24,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:25:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:24,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ad0d190b25b94785be7f2bb58c709b28 2024-11-14T07:25:24,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/7e59e05d73124121a95726c760eed1cc is 50, key is test_row_0/C:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742025_1201 (size=12151) 2024-11-14T07:25:24,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:25:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:24,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:24,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569184955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569184955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569184955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569184955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569184955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:25:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,107 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T07:25:25,107 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T07:25:25,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:25:25,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:25,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:25,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/7e59e05d73124121a95726c760eed1cc 2024-11-14T07:25:25,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3adbe23e9c66458bb82fdb20c002a015 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015 2024-11-14T07:25:25,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015, entries=300, sequenceid=186, filesize=55.7 K 2024-11-14T07:25:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/ad0d190b25b94785be7f2bb58c709b28 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28 2024-11-14T07:25:25,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T07:25:25,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/7e59e05d73124121a95726c760eed1cc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc 2024-11-14T07:25:25,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T07:25:25,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c7069097a7b68031cda66e228521c56e in 902ms, sequenceid=186, compaction requested=true 2024-11-14T07:25:25,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:25,208 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:25,209 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:25,210 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:25,210 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:25,210 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,210 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=116.8 K 2024-11-14T07:25:25,210 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,210 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015] 2024-11-14T07:25:25,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:25,211 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8105df8523a34c568b29035b69f19288, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121828 2024-11-14T07:25:25,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:25,211 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,211 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a0fa28d7d8594239be997acb479b1368, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.9 K 2024-11-14T07:25:25,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a0fa28d7d8594239be997acb479b1368, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121828 2024-11-14T07:25:25,211 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4337256b6ee4ec88912a2fe786d6f44, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731569122467 2024-11-14T07:25:25,212 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3adbe23e9c66458bb82fdb20c002a015, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123672 2024-11-14T07:25:25,212 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf7e345369a46ba8433d1e601ddbdaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731569122467 2024-11-14T07:25:25,212 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0d190b25b94785be7f2bb58c709b28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123679 2024-11-14T07:25:25,221 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,226 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:25,226 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d63dab5710474f4396632c202a285b41 is 50, key is test_row_0/B:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:25,235 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a0eee33cf0444faeb9b59fcb797b33f2_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,238 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a0eee33cf0444faeb9b59fcb797b33f2_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,238 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a0eee33cf0444faeb9b59fcb797b33f2_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742027_1203 (size=4469) 2024-11-14T07:25:25,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742026_1202 (size=12595) 2024-11-14T07:25:25,253 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d63dab5710474f4396632c202a285b41 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d63dab5710474f4396632c202a285b41 2024-11-14T07:25:25,258 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into d63dab5710474f4396632c202a285b41(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:25,258 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:25,258 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569125209; duration=0sec 2024-11-14T07:25:25,259 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:25,259 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:25,259 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:25,260 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:25,260 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:25,260 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,260 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6c8166abe60b491aa524c7df9c7e5cae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=35.9 K 2024-11-14T07:25:25,261 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8166abe60b491aa524c7df9c7e5cae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731569121828 2024-11-14T07:25:25,261 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd01df466bc48d99710ee14e67d63fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731569122467 2024-11-14T07:25:25,261 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e59e05d73124121a95726c760eed1cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123679 2024-11-14T07:25:25,269 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:25,270 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/c75decb9bb3b4eabac1f9a3a36d1ece7 is 50, key is test_row_0/C:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:25,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742028_1204 (size=12595) 2024-11-14T07:25:25,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T07:25:25,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T07:25:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a1b787e97a7541c09d5d94efff79e529_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569124323/Put/seqid=0 2024-11-14T07:25:25,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742029_1205 (size=12304) 2024-11-14T07:25:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:25,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a1b787e97a7541c09d5d94efff79e529_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a1b787e97a7541c09d5d94efff79e529_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f is 175, key is test_row_0/A:col10/1731569124323/Put/seqid=0 2024-11-14T07:25:25,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742030_1206 (size=31105) 2024-11-14T07:25:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:25,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569185496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569185499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569185500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569185501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569185500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569185604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569185615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569185615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569185615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569185615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,648 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#173 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:25,648 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/373df2defb6c4ba3b692b4820683eac8 is 175, key is test_row_0/A:col10/1731569124306/Put/seqid=0 2024-11-14T07:25:25,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742031_1207 (size=31549) 2024-11-14T07:25:25,682 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/c75decb9bb3b4eabac1f9a3a36d1ece7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/c75decb9bb3b4eabac1f9a3a36d1ece7 2024-11-14T07:25:25,687 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into c75decb9bb3b4eabac1f9a3a36d1ece7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:25,687 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:25,687 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569125209; duration=0sec 2024-11-14T07:25:25,687 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:25,687 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:25,759 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f 2024-11-14T07:25:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d229bf0dd0cd4eb78e18a27c218afcdf is 50, key is test_row_0/B:col10/1731569124323/Put/seqid=0 2024-11-14T07:25:25,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742032_1208 (size=12151) 2024-11-14T07:25:25,772 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d229bf0dd0cd4eb78e18a27c218afcdf 2024-11-14T07:25:25,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6f2c83aba6364c7e804c4f5f0fa8010f is 50, key is test_row_0/C:col10/1731569124323/Put/seqid=0 2024-11-14T07:25:25,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742033_1209 (size=12151) 2024-11-14T07:25:25,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6f2c83aba6364c7e804c4f5f0fa8010f 2024-11-14T07:25:25,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f 2024-11-14T07:25:25,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, entries=150, sequenceid=202, filesize=30.4 K 2024-11-14T07:25:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/d229bf0dd0cd4eb78e18a27c218afcdf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf 2024-11-14T07:25:25,801 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T07:25:25,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/6f2c83aba6364c7e804c4f5f0fa8010f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f 2024-11-14T07:25:25,808 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T07:25:25,809 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for c7069097a7b68031cda66e228521c56e in 478ms, sequenceid=202, compaction requested=false 2024-11-14T07:25:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T07:25:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T07:25:25,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T07:25:25,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2440 sec 2024-11-14T07:25:25,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2490 sec 2024-11-14T07:25:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:25,816 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:25,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:25:25,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111492da87f7219a4a87a99c25d40b4aaaa9_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:25,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742034_1210 (size=14794) 2024-11-14T07:25:25,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:25,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111492da87f7219a4a87a99c25d40b4aaaa9_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492da87f7219a4a87a99c25d40b4aaaa9_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:25,838 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/15e9578845c84b8691fe1d28de4cb9f5, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:25,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/15e9578845c84b8691fe1d28de4cb9f5 is 175, key is test_row_0/A:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569185832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569185832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742035_1211 (size=39749) 2024-11-14T07:25:25,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569185836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569185838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569185840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/15e9578845c84b8691fe1d28de4cb9f5 2024-11-14T07:25:25,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c78d5686b4a94205a59a28fdef680f17 is 50, key is test_row_0/B:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:25,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742036_1212 (size=12151) 2024-11-14T07:25:25,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c78d5686b4a94205a59a28fdef680f17 2024-11-14T07:25:25,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/11350fda380e43d2ad8cd49415c56046 is 50, key is test_row_0/C:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:25,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742037_1213 (size=12151) 2024-11-14T07:25:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569185944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569185944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569185955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569185955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569185955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,058 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/373df2defb6c4ba3b692b4820683eac8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8 2024-11-14T07:25:26,063 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 373df2defb6c4ba3b692b4820683eac8(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:26,063 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:26,063 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569125208; duration=0sec 2024-11-14T07:25:26,063 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:26,063 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569186155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569186155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569186164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569186164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569186164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/11350fda380e43d2ad8cd49415c56046 2024-11-14T07:25:26,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/15e9578845c84b8691fe1d28de4cb9f5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5 2024-11-14T07:25:26,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5, entries=200, sequenceid=226, filesize=38.8 K 2024-11-14T07:25:26,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c78d5686b4a94205a59a28fdef680f17 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17 2024-11-14T07:25:26,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T07:25:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/11350fda380e43d2ad8cd49415c56046 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046 2024-11-14T07:25:26,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T07:25:26,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c7069097a7b68031cda66e228521c56e in 490ms, sequenceid=226, compaction requested=true 2024-11-14T07:25:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:26,306 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:26,306 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:26,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:26,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:26,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:26,308 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:26,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:26,308 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:26,308 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,308 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,308 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=100.0 K 2024-11-14T07:25:26,308 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d63dab5710474f4396632c202a285b41, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=36.0 K 2024-11-14T07:25:26,308 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5] 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d63dab5710474f4396632c202a285b41, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123679 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 373df2defb6c4ba3b692b4820683eac8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123679 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d229bf0dd0cd4eb78e18a27c218afcdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731569124322 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731569124322 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c78d5686b4a94205a59a28fdef680f17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:26,309 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e9578845c84b8691fe1d28de4cb9f5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:26,318 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:26,318 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/80766afcab734181842633257e630aea is 50, key is test_row_0/B:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:26,321 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:26,337 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111467c5cee75d8d4842badc8f48b5a327f3_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:26,338 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111467c5cee75d8d4842badc8f48b5a327f3_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:26,338 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111467c5cee75d8d4842badc8f48b5a327f3_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742038_1214 (size=12697) 2024-11-14T07:25:26,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742039_1215 (size=4469) 2024-11-14T07:25:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:26,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:25:26,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114763971adfef04e11b800ecb21fe47342_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569126465/Put/seqid=0 2024-11-14T07:25:26,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742040_1216 (size=14794) 2024-11-14T07:25:26,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:26,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114763971adfef04e11b800ecb21fe47342_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114763971adfef04e11b800ecb21fe47342_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:26,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bedd6a01897e4d7e9eb0131a320921dd, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:26,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bedd6a01897e4d7e9eb0131a320921dd is 175, key is test_row_0/A:col10/1731569126465/Put/seqid=0 2024-11-14T07:25:26,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742041_1217 (size=39749) 2024-11-14T07:25:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569186494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569186494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569186495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569186497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569186498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569186608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569186608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569186608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569186615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569186615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T07:25:26,715 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:26,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T07:25:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:25:26,719 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:26,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:26,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:26,751 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/80766afcab734181842633257e630aea as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/80766afcab734181842633257e630aea 2024-11-14T07:25:26,752 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:26,753 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/2c582d47bf534ab98b56e7758a224202 is 175, key is test_row_0/A:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:26,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742042_1218 (size=31651) 2024-11-14T07:25:26,760 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into 80766afcab734181842633257e630aea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:26,760 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:26,760 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569126306; duration=0sec 2024-11-14T07:25:26,760 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:26,760 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:26,760 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:26,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:26,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:26,763 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,763 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/c75decb9bb3b4eabac1f9a3a36d1ece7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=36.0 K 2024-11-14T07:25:26,763 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c75decb9bb3b4eabac1f9a3a36d1ece7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731569123679 2024-11-14T07:25:26,764 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2c83aba6364c7e804c4f5f0fa8010f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731569124322 2024-11-14T07:25:26,764 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 11350fda380e43d2ad8cd49415c56046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:26,766 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/2c582d47bf534ab98b56e7758a224202 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202 2024-11-14T07:25:26,775 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 2c582d47bf534ab98b56e7758a224202(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:26,775 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:26,775 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569126306; duration=0sec 2024-11-14T07:25:26,776 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:26,776 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:26,780 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:26,780 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/38bdfa206e6f4ae2b23a41616b51c0d1 is 50, key is test_row_0/C:col10/1731569125815/Put/seqid=0 2024-11-14T07:25:26,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742043_1219 (size=12697) 2024-11-14T07:25:26,798 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/38bdfa206e6f4ae2b23a41616b51c0d1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/38bdfa206e6f4ae2b23a41616b51c0d1 2024-11-14T07:25:26,804 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 38bdfa206e6f4ae2b23a41616b51c0d1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:26,804 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:26,804 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569126306; duration=0sec 2024-11-14T07:25:26,804 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:26,804 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569186814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569186814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569186814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:25:26,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569186825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569186825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:26,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:25:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:26,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:26,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bedd6a01897e4d7e9eb0131a320921dd 2024-11-14T07:25:26,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cd6d71a303904d43bce3a241af7d726b is 50, key is test_row_0/B:col10/1731569126465/Put/seqid=0 2024-11-14T07:25:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742044_1220 (size=12151) 2024-11-14T07:25:26,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cd6d71a303904d43bce3a241af7d726b 2024-11-14T07:25:26,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/39f1097b67114e62bc5e51f4e40349ca is 50, key is test_row_0/C:col10/1731569126465/Put/seqid=0 2024-11-14T07:25:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742045_1221 (size=12151) 2024-11-14T07:25:26,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/39f1097b67114e62bc5e51f4e40349ca 2024-11-14T07:25:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bedd6a01897e4d7e9eb0131a320921dd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd 2024-11-14T07:25:26,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd, entries=200, sequenceid=242, filesize=38.8 K 2024-11-14T07:25:26,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cd6d71a303904d43bce3a241af7d726b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b 2024-11-14T07:25:26,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b, entries=150, sequenceid=242, filesize=11.9 K 2024-11-14T07:25:26,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/39f1097b67114e62bc5e51f4e40349ca as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca 2024-11-14T07:25:26,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca, entries=150, sequenceid=242, filesize=11.9 K 2024-11-14T07:25:26,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c7069097a7b68031cda66e228521c56e in 480ms, sequenceid=242, compaction requested=false 2024-11-14T07:25:26,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:27,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T07:25:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:27,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T07:25:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:25:27,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144f788c0683954137acc221b53d45bbc7_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569126496/Put/seqid=0 2024-11-14T07:25:27,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742046_1222 (size=12454) 2024-11-14T07:25:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:27,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144f788c0683954137acc221b53d45bbc7_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144f788c0683954137acc221b53d45bbc7_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:27,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a79f3272a6b44d308b7687182e3156c3, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a79f3272a6b44d308b7687182e3156c3 is 175, key is test_row_0/A:col10/1731569126496/Put/seqid=0 2024-11-14T07:25:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742047_1223 (size=31255) 2024-11-14T07:25:27,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a79f3272a6b44d308b7687182e3156c3 2024-11-14T07:25:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5c002f3452624c2abb62e27e6dfa68eb is 50, key is test_row_0/B:col10/1731569126496/Put/seqid=0 2024-11-14T07:25:27,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742048_1224 (size=12301) 2024-11-14T07:25:27,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5c002f3452624c2abb62e27e6dfa68eb 2024-11-14T07:25:27,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f3f63933c08b4ef7b95006910986b754 is 50, key is test_row_0/C:col10/1731569126496/Put/seqid=0 2024-11-14T07:25:27,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742049_1225 (size=12301) 2024-11-14T07:25:27,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569187138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569187139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569187140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569187140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569187141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569187244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569187254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569187255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569187255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569187255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:25:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569187454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569187464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569187464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569187465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569187465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f3f63933c08b4ef7b95006910986b754 2024-11-14T07:25:27,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/a79f3272a6b44d308b7687182e3156c3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3 2024-11-14T07:25:27,531 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3, entries=150, sequenceid=266, filesize=30.5 K 2024-11-14T07:25:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5c002f3452624c2abb62e27e6dfa68eb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb 2024-11-14T07:25:27,537 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb, entries=150, sequenceid=266, filesize=12.0 K 2024-11-14T07:25:27,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/f3f63933c08b4ef7b95006910986b754 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754 2024-11-14T07:25:27,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754, entries=150, sequenceid=266, filesize=12.0 K 2024-11-14T07:25:27,543 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c7069097a7b68031cda66e228521c56e in 516ms, sequenceid=266, compaction requested=true 2024-11-14T07:25:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T07:25:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T07:25:27,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T07:25:27,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 825 msec 2024-11-14T07:25:27,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 831 msec 2024-11-14T07:25:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:27,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:27,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T07:25:27,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114981e061714e84f20bbf0f5631e406bf0_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569187792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569187794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742050_1226 (size=14994) 2024-11-14T07:25:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569187795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569187796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569187796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T07:25:27,854 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:27,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T07:25:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:27,856 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:27,857 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:27,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569187905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569187905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569187905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569187914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569187914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:28,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569188114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569188114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569188114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569188124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569188124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:28,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:28,207 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114981e061714e84f20bbf0f5631e406bf0_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114981e061714e84f20bbf0f5631e406bf0_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:28,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/ba29910622df438a927efff116166e9b, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:28,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/ba29910622df438a927efff116166e9b is 175, key is test_row_0/A:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:28,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742051_1227 (size=39949) 2024-11-14T07:25:28,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569188424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569188424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569188424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569188434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569188435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:28,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/ba29910622df438a927efff116166e9b 2024-11-14T07:25:28,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/01ca8d0c0cfc4960bc65527a0efd1809 is 50, key is test_row_0/B:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742052_1228 (size=12301) 2024-11-14T07:25:28,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:28,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:28,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569188934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569188935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569188935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569188944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569188944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:29,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/01ca8d0c0cfc4960bc65527a0efd1809 2024-11-14T07:25:29,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/536d50af3e594c198dd7607bcd673db8 is 50, key is test_row_0/C:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:29,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742053_1229 (size=12301) 2024-11-14T07:25:29,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:29,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:29,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:29,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/536d50af3e594c198dd7607bcd673db8 2024-11-14T07:25:29,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/ba29910622df438a927efff116166e9b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b 2024-11-14T07:25:29,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b, entries=200, sequenceid=282, filesize=39.0 K 2024-11-14T07:25:29,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/01ca8d0c0cfc4960bc65527a0efd1809 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809 2024-11-14T07:25:29,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809, entries=150, sequenceid=282, filesize=12.0 K 2024-11-14T07:25:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/536d50af3e594c198dd7607bcd673db8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8 2024-11-14T07:25:29,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8, entries=150, sequenceid=282, filesize=12.0 K 2024-11-14T07:25:29,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c7069097a7b68031cda66e228521c56e in 1714ms, sequenceid=282, compaction requested=true 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:29,482 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:29,482 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:29,484 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:29,484 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:29,484 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,485 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=139.3 K 2024-11-14T07:25:29,485 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,485 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b] 2024-11-14T07:25:29,485 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:29,485 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:29,485 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c582d47bf534ab98b56e7758a224202, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:29,485 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,485 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/80766afcab734181842633257e630aea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=48.3 K 2024-11-14T07:25:29,485 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bedd6a01897e4d7e9eb0131a320921dd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731569125829 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 80766afcab734181842633257e630aea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79f3272a6b44d308b7687182e3156c3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569126490 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6d71a303904d43bce3a241af7d726b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731569125838 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba29910622df438a927efff116166e9b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c002f3452624c2abb62e27e6dfa68eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569126490 2024-11-14T07:25:29,486 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 01ca8d0c0cfc4960bc65527a0efd1809, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:29,493 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:29,495 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:29,495 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148af45c208c814686a6df9ac3f96b02f5_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:29,495 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/8189d25897534e11a7889bb7d8373e42 is 50, key is test_row_0/B:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:29,498 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148af45c208c814686a6df9ac3f96b02f5_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:29,498 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148af45c208c814686a6df9ac3f96b02f5_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:29,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742054_1230 (size=12983) 2024-11-14T07:25:29,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742055_1231 (size=4469) 2024-11-14T07:25:29,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T07:25:29,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T07:25:29,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cdec0efa39048a2992d04a0c0cd95f8_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569127793/Put/seqid=0 2024-11-14T07:25:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742056_1232 (size=12454) 2024-11-14T07:25:29,903 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#195 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:29,904 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f9c0918650ee409e8e6e967fd1d73261 is 175, key is test_row_0/A:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:29,911 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/8189d25897534e11a7889bb7d8373e42 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8189d25897534e11a7889bb7d8373e42 2024-11-14T07:25:29,918 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into 8189d25897534e11a7889bb7d8373e42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:29,918 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:29,918 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=12, startTime=1731569129482; duration=0sec 2024-11-14T07:25:29,919 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:29,919 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:29,919 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:29,921 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:29,921 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:29,921 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:29,921 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/38bdfa206e6f4ae2b23a41616b51c0d1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=48.3 K 2024-11-14T07:25:29,921 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bdfa206e6f4ae2b23a41616b51c0d1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569125498 2024-11-14T07:25:29,922 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f1097b67114e62bc5e51f4e40349ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731569125838 2024-11-14T07:25:29,922 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f63933c08b4ef7b95006910986b754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569126490 2024-11-14T07:25:29,924 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 536d50af3e594c198dd7607bcd673db8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:29,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742057_1233 (size=31937) 2024-11-14T07:25:29,943 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:29,943 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/3a5015f149de4942963ad38842269f53 is 50, key is test_row_0/C:col10/1731569127767/Put/seqid=0 2024-11-14T07:25:29,945 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f9c0918650ee409e8e6e967fd1d73261 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261 2024-11-14T07:25:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:29,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:29,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742058_1234 (size=12983) 2024-11-14T07:25:29,957 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into f9c0918650ee409e8e6e967fd1d73261(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:29,957 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:29,957 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=12, startTime=1731569129482; duration=0sec 2024-11-14T07:25:29,958 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:29,958 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:29,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149cdec0efa39048a2992d04a0c0cd95f8_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdec0efa39048a2992d04a0c0cd95f8_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:29,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f5bec57a9cea4d8380d18fddbaab70a8, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f5bec57a9cea4d8380d18fddbaab70a8 is 175, key is test_row_0/A:col10/1731569127793/Put/seqid=0 2024-11-14T07:25:29,967 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/3a5015f149de4942963ad38842269f53 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/3a5015f149de4942963ad38842269f53 2024-11-14T07:25:29,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:29,977 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 3a5015f149de4942963ad38842269f53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:29,977 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569189970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:29,977 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=12, startTime=1731569129482; duration=0sec 2024-11-14T07:25:29,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569189972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:29,978 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:29,978 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:29,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569189972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569189974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742059_1235 (size=31255) 2024-11-14T07:25:29,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f5bec57a9cea4d8380d18fddbaab70a8 2024-11-14T07:25:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569189978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5897bca8813248b6934ca5cdada39693 is 50, key is test_row_0/B:col10/1731569127793/Put/seqid=0 2024-11-14T07:25:30,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742060_1236 (size=12301) 2024-11-14T07:25:30,037 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5897bca8813248b6934ca5cdada39693 2024-11-14T07:25:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/89d8cf8085d34ba2af04471158c4d02c is 50, key is test_row_0/C:col10/1731569127793/Put/seqid=0 2024-11-14T07:25:30,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742061_1237 (size=12301) 2024-11-14T07:25:30,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569190085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569190085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569190085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569190085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569190095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,139 INFO [master/c7aa6e2790b3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T07:25:30,139 INFO [master/c7aa6e2790b3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T07:25:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569190294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569190294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569190294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569190294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569190304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/89d8cf8085d34ba2af04471158c4d02c 2024-11-14T07:25:30,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/f5bec57a9cea4d8380d18fddbaab70a8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8 2024-11-14T07:25:30,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8, entries=150, sequenceid=302, filesize=30.5 K 2024-11-14T07:25:30,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/5897bca8813248b6934ca5cdada39693 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693 2024-11-14T07:25:30,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T07:25:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/89d8cf8085d34ba2af04471158c4d02c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c 2024-11-14T07:25:30,497 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T07:25:30,498 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c7069097a7b68031cda66e228521c56e in 953ms, sequenceid=302, compaction requested=false 2024-11-14T07:25:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T07:25:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T07:25:30,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T07:25:30,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-14T07:25:30,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6480 sec 2024-11-14T07:25:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:30,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:30,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:30,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf4bd9516d994747a486a244d5281297_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742062_1238 (size=17534) 2024-11-14T07:25:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569190628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569190631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569190632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569190632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569190632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569190735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569190744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569190744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569190754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569190754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569190944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569190954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569190954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569190964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569190964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:31,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf4bd9516d994747a486a244d5281297_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4bd9516d994747a486a244d5281297_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:31,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7b2ad4ba05b14d84ac1214951ae0e3de, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:31,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7b2ad4ba05b14d84ac1214951ae0e3de is 175, key is test_row_0/A:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:31,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742063_1239 (size=48639) 2024-11-14T07:25:31,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569191255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569191264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569191264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569191275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569191275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,432 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7b2ad4ba05b14d84ac1214951ae0e3de 2024-11-14T07:25:31,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/4257b9c1769742f8a3659cc80958f15f is 50, key is test_row_0/B:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:31,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742064_1240 (size=12301) 2024-11-14T07:25:31,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/4257b9c1769742f8a3659cc80958f15f 2024-11-14T07:25:31,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/dcbdbeeb78594589a7ae270c8f17a546 is 50, key is test_row_0/C:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:31,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742065_1241 (size=12301) 2024-11-14T07:25:31,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569191765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569191774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569191774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569191785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569191785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:31,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/dcbdbeeb78594589a7ae270c8f17a546 2024-11-14T07:25:31,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7b2ad4ba05b14d84ac1214951ae0e3de as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de 2024-11-14T07:25:31,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de, entries=250, sequenceid=325, filesize=47.5 K 2024-11-14T07:25:31,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/4257b9c1769742f8a3659cc80958f15f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f 2024-11-14T07:25:31,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T07:25:31,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/dcbdbeeb78594589a7ae270c8f17a546 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546 2024-11-14T07:25:31,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T07:25:31,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for c7069097a7b68031cda66e228521c56e in 1361ms, sequenceid=325, compaction requested=true 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:31,968 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T07:25:31,969 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:31,969 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:31,970 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:31,970 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:31,970 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=109.2 K 2024-11-14T07:25:31,970 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8189d25897534e11a7889bb7d8373e42, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=36.7 K 2024-11-14T07:25:31,970 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de] 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8189d25897534e11a7889bb7d8373e42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:31,970 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c0918650ee409e8e6e967fd1d73261, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:31,971 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bec57a9cea4d8380d18fddbaab70a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731569127792 2024-11-14T07:25:31,971 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5897bca8813248b6934ca5cdada39693, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731569127792 2024-11-14T07:25:31,972 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2ad4ba05b14d84ac1214951ae0e3de, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:31,972 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4257b9c1769742f8a3659cc80958f15f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:31,984 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:31,988 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114394c03f3aaf340bea95e79d6382c726c_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:31,990 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114394c03f3aaf340bea95e79d6382c726c_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:31,991 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114394c03f3aaf340bea95e79d6382c726c_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:32,003 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:32,004 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/aa2d3657c625498fad74b0bcbfa1f533 is 50, key is test_row_0/B:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T07:25:32,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:32,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-14T07:25:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:32,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:32,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:32,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:32,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742066_1242 (size=4469) 2024-11-14T07:25:32,041 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#203 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:32,044 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7baa51332b5f47f68b4b3069b6c3cf40 is 175, key is test_row_0/A:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:32,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742067_1243 (size=13085) 2024-11-14T07:25:32,098 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/aa2d3657c625498fad74b0bcbfa1f533 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/aa2d3657c625498fad74b0bcbfa1f533 2024-11-14T07:25:32,104 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into aa2d3657c625498fad74b0bcbfa1f533(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:32,104 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:32,104 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569131968; duration=0sec 2024-11-14T07:25:32,104 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:32,104 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:32,104 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:32,105 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:32,105 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:32,105 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:32,106 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/3a5015f149de4942963ad38842269f53, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=36.7 K 2024-11-14T07:25:32,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5015f149de4942963ad38842269f53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731569127138 2024-11-14T07:25:32,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d8cf8085d34ba2af04471158c4d02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731569127792 2024-11-14T07:25:32,107 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcbdbeeb78594589a7ae270c8f17a546, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742068_1244 (size=32039) 2024-11-14T07:25:32,130 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:32,131 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/e60d50d0e6914ffbb46e819849dc15f4 is 50, key is test_row_0/C:col10/1731569129962/Put/seqid=0 2024-11-14T07:25:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:32,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742069_1245 (size=13085) 2024-11-14T07:25:32,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T07:25:32,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:32,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cca413725ef14f01bee1db57dac9d566_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569130618/Put/seqid=0 2024-11-14T07:25:32,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742070_1246 (size=12454) 2024-11-14T07:25:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:32,513 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/7baa51332b5f47f68b4b3069b6c3cf40 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40 2024-11-14T07:25:32,520 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 7baa51332b5f47f68b4b3069b6c3cf40(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:32,520 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:32,520 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569131968; duration=0sec 2024-11-14T07:25:32,520 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:32,520 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:32,573 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/e60d50d0e6914ffbb46e819849dc15f4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/e60d50d0e6914ffbb46e819849dc15f4 2024-11-14T07:25:32,584 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into e60d50d0e6914ffbb46e819849dc15f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:32,584 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:32,584 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569131968; duration=0sec 2024-11-14T07:25:32,585 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:32,585 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:32,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cca413725ef14f01bee1db57dac9d566_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cca413725ef14f01bee1db57dac9d566_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/478e26d7d0d14364a99dbf7e8946aff0, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:32,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/478e26d7d0d14364a99dbf7e8946aff0 is 175, key is test_row_0/A:col10/1731569130618/Put/seqid=0 2024-11-14T07:25:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742071_1247 (size=31255) 2024-11-14T07:25:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569192804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569192804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569192806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,811 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569192811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569192811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569192914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569192914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569192925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569192925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569193124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569193124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569193135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569193135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/478e26d7d0d14364a99dbf7e8946aff0 2024-11-14T07:25:33,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a369b6538f7e4666bdbac3aef64f0fae is 50, key is test_row_0/B:col10/1731569130618/Put/seqid=0 2024-11-14T07:25:33,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742072_1248 (size=12301) 2024-11-14T07:25:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:33,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569193434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569193434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569193445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569193445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,554 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a369b6538f7e4666bdbac3aef64f0fae 2024-11-14T07:25:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/df868d502ac24b87aed921f87f990a51 is 50, key is test_row_0/C:col10/1731569130618/Put/seqid=0 2024-11-14T07:25:33,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742073_1249 (size=12301) 2024-11-14T07:25:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569193944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569193944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569193955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569193955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:33,968 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/df868d502ac24b87aed921f87f990a51 2024-11-14T07:25:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/478e26d7d0d14364a99dbf7e8946aff0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0 2024-11-14T07:25:33,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0, entries=150, sequenceid=344, filesize=30.5 K 2024-11-14T07:25:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a369b6538f7e4666bdbac3aef64f0fae as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae 2024-11-14T07:25:33,981 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae, entries=150, sequenceid=344, filesize=12.0 K 2024-11-14T07:25:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/df868d502ac24b87aed921f87f990a51 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51 2024-11-14T07:25:33,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51, entries=150, sequenceid=344, filesize=12.0 K 2024-11-14T07:25:33,987 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for c7069097a7b68031cda66e228521c56e in 1803ms, sequenceid=344, compaction requested=false 2024-11-14T07:25:33,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:33,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:33,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-14T07:25:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-14T07:25:33,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T07:25:33,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9590 sec 2024-11-14T07:25:33,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9640 sec 2024-11-14T07:25:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T07:25:34,175 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:34,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T07:25:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:34,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:34,181 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:34,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:34,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T07:25:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:34,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T07:25:34,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148b49ffa9b2324a0fade15b07b545a6ef_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569132798/Put/seqid=0 2024-11-14T07:25:34,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742074_1250 (size=12454) 2024-11-14T07:25:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:34,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148b49ffa9b2324a0fade15b07b545a6ef_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148b49ffa9b2324a0fade15b07b545a6ef_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:34,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bf2334dd75634f6783d31399f9894946, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:34,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bf2334dd75634f6783d31399f9894946 is 175, key is test_row_0/A:col10/1731569132798/Put/seqid=0 2024-11-14T07:25:34,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742075_1251 (size=31255) 2024-11-14T07:25:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:34,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569194867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569194954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569194954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,957 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:34,958 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569194964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569194964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,971 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:34,971 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569194975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:35,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bf2334dd75634f6783d31399f9894946 2024-11-14T07:25:35,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569195184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf47eeb428d54a118423cfa4c86d4689 is 50, key is test_row_0/B:col10/1731569132798/Put/seqid=0 2024-11-14T07:25:35,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742076_1252 (size=12301) 2024-11-14T07:25:35,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf47eeb428d54a118423cfa4c86d4689 2024-11-14T07:25:35,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/d1356b7751254d64becc0c4648dc00f2 is 50, key is test_row_0/C:col10/1731569132798/Put/seqid=0 2024-11-14T07:25:35,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742077_1253 (size=12301) 2024-11-14T07:25:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:35,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569195495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:35,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/d1356b7751254d64becc0c4648dc00f2 2024-11-14T07:25:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/bf2334dd75634f6783d31399f9894946 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946 2024-11-14T07:25:35,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946, entries=150, sequenceid=365, filesize=30.5 K 2024-11-14T07:25:35,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/cf47eeb428d54a118423cfa4c86d4689 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689 2024-11-14T07:25:35,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T07:25:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/d1356b7751254d64becc0c4648dc00f2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2 2024-11-14T07:25:35,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T07:25:35,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c7069097a7b68031cda66e228521c56e in 1314ms, sequenceid=365, compaction requested=true 2024-11-14T07:25:35,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:35,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:35,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T07:25:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T07:25:35,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T07:25:35,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-11-14T07:25:35,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4780 sec 2024-11-14T07:25:36,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:36,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:36,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144bee99cc60fd4588ba43438d3e1de92c_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742078_1254 (size=14994) 2024-11-14T07:25:36,087 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:36,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144bee99cc60fd4588ba43438d3e1de92c_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bee99cc60fd4588ba43438d3e1de92c_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:36,096 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/14bee1d0256a4f62b7207dfb8bb4989c, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:36,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/14bee1d0256a4f62b7207dfb8bb4989c is 175, key is test_row_0/A:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:36,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742079_1255 (size=39949) 2024-11-14T07:25:36,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/14bee1d0256a4f62b7207dfb8bb4989c 2024-11-14T07:25:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569196119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/eeb865318530463e8ed5bc640912ebfc is 50, key is test_row_0/B:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:36,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742080_1256 (size=12301) 2024-11-14T07:25:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569196235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T07:25:36,325 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:36,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-14T07:25:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:36,327 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:36,327 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:36,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569196445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T07:25:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:36,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/eeb865318530463e8ed5bc640912ebfc 2024-11-14T07:25:36,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/ef00924be31d41e6bd23a036783952dc is 50, key is test_row_0/C:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:36,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742081_1257 (size=12301) 2024-11-14T07:25:36,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T07:25:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569196754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T07:25:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T07:25:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:36,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45172 deadline: 1731569196964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,965 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/ef00924be31d41e6bd23a036783952dc 2024-11-14T07:25:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45170 deadline: 1731569196984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45124 deadline: 1731569196984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,990 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:36,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,991 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:36,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:36,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/14bee1d0256a4f62b7207dfb8bb4989c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c 2024-11-14T07:25:36,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c, entries=200, sequenceid=383, filesize=39.0 K 2024-11-14T07:25:36,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/eeb865318530463e8ed5bc640912ebfc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc 2024-11-14T07:25:36,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45126 deadline: 1731569196995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:36,999 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in c7069097a7b68031cda66e228521c56e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:25:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:37,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T07:25:37,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/ef00924be31d41e6bd23a036783952dc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc 2024-11-14T07:25:37,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T07:25:37,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c7069097a7b68031cda66e228521c56e in 995ms, sequenceid=383, compaction requested=true 2024-11-14T07:25:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:37,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:37,013 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:37,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:37,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:37,013 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:37,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:37,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:37,014 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:37,014 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:37,014 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=131.3 K 2024-11-14T07:25:37,014 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/aa2d3657c625498fad74b0bcbfa1f533, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=48.8 K 2024-11-14T07:25:37,014 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c] 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2d3657c625498fad74b0bcbfa1f533, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7baa51332b5f47f68b4b3069b6c3cf40, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:37,014 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a369b6538f7e4666bdbac3aef64f0fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731569130618 2024-11-14T07:25:37,015 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478e26d7d0d14364a99dbf7e8946aff0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731569130618 2024-11-14T07:25:37,015 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting cf47eeb428d54a118423cfa4c86d4689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731569132798 2024-11-14T07:25:37,015 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf2334dd75634f6783d31399f9894946, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731569132798 2024-11-14T07:25:37,015 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb865318530463e8ed5bc640912ebfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:37,015 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14bee1d0256a4f62b7207dfb8bb4989c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:37,022 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:37,023 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:37,023 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/8221976732364dc39414d4ec6de80a66 is 50, key is test_row_0/B:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:37,023 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114c7d1f39769384618b83b28b1228c4fb7_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:37,025 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114c7d1f39769384618b83b28b1228c4fb7_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:37,025 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c7d1f39769384618b83b28b1228c4fb7_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:37,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742082_1258 (size=13221) 2024-11-14T07:25:37,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742083_1259 (size=4469) 2024-11-14T07:25:37,033 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#216 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:37,034 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/86a725eef236444a8d6f8eda04e13763 is 175, key is test_row_0/A:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:37,035 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/8221976732364dc39414d4ec6de80a66 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8221976732364dc39414d4ec6de80a66 2024-11-14T07:25:37,042 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into 8221976732364dc39414d4ec6de80a66(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:37,042 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:37,042 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=12, startTime=1731569137013; duration=0sec 2024-11-14T07:25:37,042 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:37,042 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:37,042 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:25:37,044 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:25:37,044 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:37,044 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:37,044 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/e60d50d0e6914ffbb46e819849dc15f4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=48.8 K 2024-11-14T07:25:37,044 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e60d50d0e6914ffbb46e819849dc15f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731569129962 2024-11-14T07:25:37,045 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting df868d502ac24b87aed921f87f990a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731569130618 2024-11-14T07:25:37,045 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d1356b7751254d64becc0c4648dc00f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731569132798 2024-11-14T07:25:37,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742084_1260 (size=32175) 2024-11-14T07:25:37,045 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ef00924be31d41e6bd23a036783952dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:37,051 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/86a725eef236444a8d6f8eda04e13763 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763 2024-11-14T07:25:37,055 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:37,056 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/fd04739e7f4548d3b7abf54a6257f94d is 50, key is test_row_0/C:col10/1731569134862/Put/seqid=0 2024-11-14T07:25:37,059 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 86a725eef236444a8d6f8eda04e13763(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:37,059 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:37,059 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=12, startTime=1731569137012; duration=0sec 2024-11-14T07:25:37,059 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:37,060 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:37,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742085_1261 (size=13221) 2024-11-14T07:25:37,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T07:25:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:37,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144818c3639860406c945ccfe3b2a82c93_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:37,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742086_1262 (size=12454) 2024-11-14T07:25:37,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:37,104 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144818c3639860406c945ccfe3b2a82c93_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144818c3639860406c945ccfe3b2a82c93_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3d95e340c066420bac7f82862b2e9615, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:37,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3d95e340c066420bac7f82862b2e9615 is 175, key is test_row_0/A:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:37,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742087_1263 (size=31255) 2024-11-14T07:25:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:37,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. as already flushing 2024-11-14T07:25:37,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569197315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:37,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569197424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:37,465 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/fd04739e7f4548d3b7abf54a6257f94d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/fd04739e7f4548d3b7abf54a6257f94d 2024-11-14T07:25:37,469 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into fd04739e7f4548d3b7abf54a6257f94d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:37,469 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:37,469 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=12, startTime=1731569137013; duration=0sec 2024-11-14T07:25:37,470 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:37,470 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:37,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3d95e340c066420bac7f82862b2e9615 2024-11-14T07:25:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a40bc66d995241af90086739327ba465 is 50, key is test_row_0/B:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742088_1264 (size=12301) 2024-11-14T07:25:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569197635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:37,931 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a40bc66d995241af90086739327ba465 2024-11-14T07:25:37,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4822351e59364b569153af7877e383a0 is 50, key is test_row_0/C:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:37,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742089_1265 (size=12301) 2024-11-14T07:25:37,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45202 deadline: 1731569197944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:25:38,110 INFO [Thread-740 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-740. 2024-11-14T07:25:38,110 DEBUG [Thread-740 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:38,110 INFO [Thread-744 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-744. 2024-11-14T07:25:38,110 DEBUG [Thread-740 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,110 DEBUG [Thread-744 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:38,110 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,110 DEBUG [Thread-740 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,110 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:38,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:38,111 INFO [Thread-738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-738. 2024-11-14T07:25:38,111 DEBUG [Thread-738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:38,111 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,111 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:38,113 INFO [Thread-742 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-742. 2024-11-14T07:25:38,113 DEBUG [Thread-742 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:38,113 DEBUG [Thread-742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,113 DEBUG [Thread-742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:38,342 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4822351e59364b569153af7877e383a0 2024-11-14T07:25:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/3d95e340c066420bac7f82862b2e9615 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615 2024-11-14T07:25:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615, entries=150, sequenceid=403, filesize=30.5 K 2024-11-14T07:25:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a40bc66d995241af90086739327ba465 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465 2024-11-14T07:25:38,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T07:25:38,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/4822351e59364b569153af7877e383a0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0 2024-11-14T07:25:38,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T07:25:38,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c7069097a7b68031cda66e228521c56e in 1274ms, sequenceid=403, compaction requested=false 2024-11-14T07:25:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T07:25:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T07:25:38,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T07:25:38,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0390 sec 2024-11-14T07:25:38,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.0430 sec 2024-11-14T07:25:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:38,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:38,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T07:25:38,456 INFO [Thread-731 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-731. 2024-11-14T07:25:38,456 DEBUG [Thread-731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:38,456 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,456 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:38,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:38,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114445623f7ecbb488683e8775e624deb30_c7069097a7b68031cda66e228521c56e is 50, key is test_row_1/A:col10/1731569138454/Put/seqid=0 2024-11-14T07:25:38,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742090_1266 (size=9914) 2024-11-14T07:25:38,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:38,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114445623f7ecbb488683e8775e624deb30_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114445623f7ecbb488683e8775e624deb30_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T07:25:38,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/25e930ef4acd4b57bc23e109e1057434, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:38,475 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:38,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/25e930ef4acd4b57bc23e109e1057434 is 175, key is test_row_1/A:col10/1731569138454/Put/seqid=0 2024-11-14T07:25:38,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742091_1267 (size=22561) 2024-11-14T07:25:38,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/25e930ef4acd4b57bc23e109e1057434 2024-11-14T07:25:38,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a7121651de904de98474fba050a4ed0a is 50, key is test_row_1/B:col10/1731569138454/Put/seqid=0 2024-11-14T07:25:38,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742092_1268 (size=9857) 2024-11-14T07:25:39,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a7121651de904de98474fba050a4ed0a 2024-11-14T07:25:39,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/97f2da44dad84389babb59c68b18f601 is 50, key is test_row_1/C:col10/1731569138454/Put/seqid=0 2024-11-14T07:25:39,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742093_1269 (size=9857) 2024-11-14T07:25:39,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/97f2da44dad84389babb59c68b18f601 2024-11-14T07:25:39,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/25e930ef4acd4b57bc23e109e1057434 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434 2024-11-14T07:25:39,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434, entries=100, sequenceid=423, filesize=22.0 K 2024-11-14T07:25:39,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/a7121651de904de98474fba050a4ed0a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a 2024-11-14T07:25:39,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a, entries=100, sequenceid=423, filesize=9.6 K 2024-11-14T07:25:39,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/97f2da44dad84389babb59c68b18f601 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601 2024-11-14T07:25:39,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601, entries=100, sequenceid=423, filesize=9.6 K 2024-11-14T07:25:39,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for c7069097a7b68031cda66e228521c56e in 1276ms, sequenceid=423, compaction requested=true 2024-11-14T07:25:39,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:39,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:39,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:39,733 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7069097a7b68031cda66e228521c56e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:39,733 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/A is initiating minor compaction (all files) 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/B is initiating minor compaction (all files) 2024-11-14T07:25:39,734 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/A in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:39,734 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/B in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:39,734 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=84.0 K 2024-11-14T07:25:39,734 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8221976732364dc39414d4ec6de80a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=34.5 K 2024-11-14T07:25:39,734 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434] 2024-11-14T07:25:39,734 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8221976732364dc39414d4ec6de80a66, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:39,735 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a725eef236444a8d6f8eda04e13763, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:39,735 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a40bc66d995241af90086739327ba465, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731569136099 2024-11-14T07:25:39,735 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d95e340c066420bac7f82862b2e9615, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731569136099 2024-11-14T07:25:39,735 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a7121651de904de98474fba050a4ed0a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731569137313 2024-11-14T07:25:39,735 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25e930ef4acd4b57bc23e109e1057434, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731569137313 2024-11-14T07:25:39,744 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:39,761 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d7688eb7079f41f1945f62db3b0357cc_c7069097a7b68031cda66e228521c56e store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:39,761 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#B#compaction#224 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:39,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/9d17c3aafc754240895ad82da7dea7ac is 50, key is test_row_0/B:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:39,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742094_1270 (size=13323) 2024-11-14T07:25:39,781 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d7688eb7079f41f1945f62db3b0357cc_c7069097a7b68031cda66e228521c56e, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:39,781 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d7688eb7079f41f1945f62db3b0357cc_c7069097a7b68031cda66e228521c56e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:39,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742095_1271 (size=4469) 2024-11-14T07:25:40,175 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/9d17c3aafc754240895ad82da7dea7ac as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/9d17c3aafc754240895ad82da7dea7ac 2024-11-14T07:25:40,181 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/B of c7069097a7b68031cda66e228521c56e into 9d17c3aafc754240895ad82da7dea7ac(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:40,181 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:40,181 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/B, priority=13, startTime=1731569139732; duration=0sec 2024-11-14T07:25:40,181 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:40,182 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:B 2024-11-14T07:25:40,182 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:40,182 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:40,183 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): c7069097a7b68031cda66e228521c56e/C is initiating minor compaction (all files) 2024-11-14T07:25:40,183 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7069097a7b68031cda66e228521c56e/C in TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:40,183 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/fd04739e7f4548d3b7abf54a6257f94d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp, totalSize=34.5 K 2024-11-14T07:25:40,183 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fd04739e7f4548d3b7abf54a6257f94d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569134844 2024-11-14T07:25:40,184 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4822351e59364b569153af7877e383a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731569136099 2024-11-14T07:25:40,184 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f2da44dad84389babb59c68b18f601, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731569137313 2024-11-14T07:25:40,187 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#A#compaction#225 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:40,187 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/81470b43d968412996f97532a1941ded is 175, key is test_row_0/A:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:40,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742096_1272 (size=32384) 2024-11-14T07:25:40,196 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7069097a7b68031cda66e228521c56e#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:40,196 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/81470b43d968412996f97532a1941ded as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/81470b43d968412996f97532a1941ded 2024-11-14T07:25:40,196 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/9e52b5fbfc614c75ac49fc2f97b025a9 is 50, key is test_row_0/C:col10/1731569136099/Put/seqid=0 2024-11-14T07:25:40,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742097_1273 (size=13323) 2024-11-14T07:25:40,203 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/A of c7069097a7b68031cda66e228521c56e into 81470b43d968412996f97532a1941ded(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:40,203 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:40,203 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/A, priority=13, startTime=1731569139732; duration=0sec 2024-11-14T07:25:40,203 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:40,203 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:A 2024-11-14T07:25:40,206 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/9e52b5fbfc614c75ac49fc2f97b025a9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9e52b5fbfc614c75ac49fc2f97b025a9 2024-11-14T07:25:40,211 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7069097a7b68031cda66e228521c56e/C of c7069097a7b68031cda66e228521c56e into 9e52b5fbfc614c75ac49fc2f97b025a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:40,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7069097a7b68031cda66e228521c56e: 2024-11-14T07:25:40,211 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., storeName=c7069097a7b68031cda66e228521c56e/C, priority=13, startTime=1731569139733; duration=0sec 2024-11-14T07:25:40,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:40,211 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7069097a7b68031cda66e228521c56e:C 2024-11-14T07:25:41,007 INFO [Thread-729 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-729. 2024-11-14T07:25:41,007 DEBUG [Thread-729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:41,007 DEBUG [Thread-729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,007 DEBUG [Thread-729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:41,018 INFO [Thread-727 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-727. 2024-11-14T07:25:41,018 DEBUG [Thread-727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:41,018 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,018 DEBUG [Thread-727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:41,038 INFO [Thread-733 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-733. 2024-11-14T07:25:41,038 DEBUG [Thread-733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:41,038 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,039 INFO [Thread-735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-735. 2024-11-14T07:25:41,039 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:41,039 DEBUG [Thread-735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:25:41,039 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,039 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3890 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3783 2024-11-14T07:25:41,040 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:25:41,041 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1710 2024-11-14T07:25:41,041 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5129 rows 2024-11-14T07:25:41,041 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1714 2024-11-14T07:25:41,041 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5142 rows 2024-11-14T07:25:41,041 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:25:41,041 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:25:41,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:41,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:25:41,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:25:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:41,045 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569141045"}]},"ts":"1731569141045"} 2024-11-14T07:25:41,046 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:25:41,046 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:25:41,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:25:41,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, UNASSIGN}] 2024-11-14T07:25:41,049 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, UNASSIGN 2024-11-14T07:25:41,049 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:41,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, UNASSIGN because future has completed 2024-11-14T07:25:41,051 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:25:41,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:41,203 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:41,203 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:25:41,203 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing c7069097a7b68031cda66e228521c56e, disabling compactions & flushes 2024-11-14T07:25:41,203 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:41,203 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:41,203 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. after waiting 0 ms 2024-11-14T07:25:41,203 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:41,203 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing c7069097a7b68031cda66e228521c56e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T07:25:41,210 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148781056cf1604e55bfc542dd46ed8afb_c7069097a7b68031cda66e228521c56e is 50, key is test_row_0/A:col10/1731569141036/Put/seqid=0 2024-11-14T07:25:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742098_1274 (size=9914) 2024-11-14T07:25:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:41,614 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:41,622 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148781056cf1604e55bfc542dd46ed8afb_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148781056cf1604e55bfc542dd46ed8afb_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:41,624 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/03e681d45f4f4cd6908587b25887259c, store: [table=TestAcidGuarantees family=A region=c7069097a7b68031cda66e228521c56e] 2024-11-14T07:25:41,625 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/03e681d45f4f4cd6908587b25887259c is 175, key is test_row_0/A:col10/1731569141036/Put/seqid=0 2024-11-14T07:25:41,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742099_1275 (size=22561) 2024-11-14T07:25:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:42,032 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/03e681d45f4f4cd6908587b25887259c 2024-11-14T07:25:42,041 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c049be2d76024771814e31dbc9f10ba7 is 50, key is test_row_0/B:col10/1731569141036/Put/seqid=0 2024-11-14T07:25:42,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742100_1276 (size=9857) 2024-11-14T07:25:42,045 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c049be2d76024771814e31dbc9f10ba7 2024-11-14T07:25:42,054 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/17de501cceb140f9a96403aeb38d10af is 50, key is test_row_0/C:col10/1731569141036/Put/seqid=0 2024-11-14T07:25:42,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742101_1277 (size=9857) 2024-11-14T07:25:42,058 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/17de501cceb140f9a96403aeb38d10af 2024-11-14T07:25:42,062 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/A/03e681d45f4f4cd6908587b25887259c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/03e681d45f4f4cd6908587b25887259c 2024-11-14T07:25:42,067 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/03e681d45f4f4cd6908587b25887259c, entries=100, sequenceid=433, filesize=22.0 K 2024-11-14T07:25:42,068 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/B/c049be2d76024771814e31dbc9f10ba7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c049be2d76024771814e31dbc9f10ba7 2024-11-14T07:25:42,072 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c049be2d76024771814e31dbc9f10ba7, entries=100, sequenceid=433, filesize=9.6 K 2024-11-14T07:25:42,074 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/.tmp/C/17de501cceb140f9a96403aeb38d10af as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/17de501cceb140f9a96403aeb38d10af 2024-11-14T07:25:42,079 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/17de501cceb140f9a96403aeb38d10af, entries=100, sequenceid=433, filesize=9.6 K 2024-11-14T07:25:42,080 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c7069097a7b68031cda66e228521c56e in 877ms, sequenceid=433, compaction requested=false 2024-11-14T07:25:42,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434] to archive 2024-11-14T07:25:42,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:42,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/84e81dc12e754a0bb99aef107ef1e87f 2024-11-14T07:25:42,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/0b3579cbf66b4ad3ac740f7f010296de 2024-11-14T07:25:42,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/461527fc4a7c4fb8b5722295782d62b6 2024-11-14T07:25:42,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/307b385a13924b52b8ba82ceff4d005a 2024-11-14T07:25:42,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/651c0b2605794fb695dc0a686b91e94c 2024-11-14T07:25:42,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/779a740a0b4344c2bab12fcd4713bcea 2024-11-14T07:25:42,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/654a51a889ed4dcd85122195b96d1989 2024-11-14T07:25:42,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/094bf1cdd8cc47fa966e925b690a8829 2024-11-14T07:25:42,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bb289b9688484c3bbca02940544948bb 2024-11-14T07:25:42,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8bfc758b873d418682743e870a40b861 2024-11-14T07:25:42,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/8105df8523a34c568b29035b69f19288 2024-11-14T07:25:42,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/e4337256b6ee4ec88912a2fe786d6f44 2024-11-14T07:25:42,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3adbe23e9c66458bb82fdb20c002a015 2024-11-14T07:25:42,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/373df2defb6c4ba3b692b4820683eac8 2024-11-14T07:25:42,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a7bc30374b7c4cb8a3e9fcc2ac6f9f0f 2024-11-14T07:25:42,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/15e9578845c84b8691fe1d28de4cb9f5 2024-11-14T07:25:42,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/2c582d47bf534ab98b56e7758a224202 2024-11-14T07:25:42,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bedd6a01897e4d7e9eb0131a320921dd 2024-11-14T07:25:42,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/a79f3272a6b44d308b7687182e3156c3 2024-11-14T07:25:42,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/ba29910622df438a927efff116166e9b 2024-11-14T07:25:42,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f9c0918650ee409e8e6e967fd1d73261 2024-11-14T07:25:42,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/f5bec57a9cea4d8380d18fddbaab70a8 2024-11-14T07:25:42,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7b2ad4ba05b14d84ac1214951ae0e3de 2024-11-14T07:25:42,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/7baa51332b5f47f68b4b3069b6c3cf40 2024-11-14T07:25:42,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/478e26d7d0d14364a99dbf7e8946aff0 2024-11-14T07:25:42,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/bf2334dd75634f6783d31399f9894946 2024-11-14T07:25:42,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/14bee1d0256a4f62b7207dfb8bb4989c 2024-11-14T07:25:42,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/86a725eef236444a8d6f8eda04e13763 2024-11-14T07:25:42,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/3d95e340c066420bac7f82862b2e9615 2024-11-14T07:25:42,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/25e930ef4acd4b57bc23e109e1057434 2024-11-14T07:25:42,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d27de005e95f4a34beeef220bb18f95a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/392f175a8044494880bdfa099931a3ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a0fa28d7d8594239be997acb479b1368, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d63dab5710474f4396632c202a285b41, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/80766afcab734181842633257e630aea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8189d25897534e11a7889bb7d8373e42, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/aa2d3657c625498fad74b0bcbfa1f533, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8221976732364dc39414d4ec6de80a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a] to archive 2024-11-14T07:25:42,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:42,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ca899333058a4b33a2aa0a42ff407324 2024-11-14T07:25:42,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf034d5b59b44e2f982cae3f8bdfc857 2024-11-14T07:25:42,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d27de005e95f4a34beeef220bb18f95a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d27de005e95f4a34beeef220bb18f95a 2024-11-14T07:25:42,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5266ecb7091f46169d0018f7f967fec6 2024-11-14T07:25:42,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/23bc8e341b0443ffaddf4a63650e69a3 2024-11-14T07:25:42,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a13c5e4c965b4ddbb2921623d0fddd3d 2024-11-14T07:25:42,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/392f175a8044494880bdfa099931a3ba to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/392f175a8044494880bdfa099931a3ba 2024-11-14T07:25:42,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/883ed4e3a0554b11b7d2b88c00ad0adf 2024-11-14T07:25:42,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/f79b55b34bf24dd88d323323abc1dbe0 2024-11-14T07:25:42,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a0fa28d7d8594239be997acb479b1368 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a0fa28d7d8594239be997acb479b1368 2024-11-14T07:25:42,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/04195f54032b44bba18d7969363686af 2024-11-14T07:25:42,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/1bf7e345369a46ba8433d1e601ddbdaf 2024-11-14T07:25:42,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d63dab5710474f4396632c202a285b41 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d63dab5710474f4396632c202a285b41 2024-11-14T07:25:42,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/ad0d190b25b94785be7f2bb58c709b28 2024-11-14T07:25:42,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/d229bf0dd0cd4eb78e18a27c218afcdf 2024-11-14T07:25:42,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/80766afcab734181842633257e630aea to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/80766afcab734181842633257e630aea 2024-11-14T07:25:42,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c78d5686b4a94205a59a28fdef680f17 2024-11-14T07:25:42,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cd6d71a303904d43bce3a241af7d726b 2024-11-14T07:25:42,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5c002f3452624c2abb62e27e6dfa68eb 2024-11-14T07:25:42,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8189d25897534e11a7889bb7d8373e42 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8189d25897534e11a7889bb7d8373e42 2024-11-14T07:25:42,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/01ca8d0c0cfc4960bc65527a0efd1809 2024-11-14T07:25:42,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/5897bca8813248b6934ca5cdada39693 2024-11-14T07:25:42,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/aa2d3657c625498fad74b0bcbfa1f533 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/aa2d3657c625498fad74b0bcbfa1f533 2024-11-14T07:25:42,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/4257b9c1769742f8a3659cc80958f15f 2024-11-14T07:25:42,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a369b6538f7e4666bdbac3aef64f0fae 2024-11-14T07:25:42,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/cf47eeb428d54a118423cfa4c86d4689 2024-11-14T07:25:42,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8221976732364dc39414d4ec6de80a66 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/8221976732364dc39414d4ec6de80a66 2024-11-14T07:25:42,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/eeb865318530463e8ed5bc640912ebfc 2024-11-14T07:25:42,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a40bc66d995241af90086739327ba465 2024-11-14T07:25:42,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/a7121651de904de98474fba050a4ed0a 2024-11-14T07:25:42,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/35df91d6f7fa42ff83a9c8c0adef11fd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9a371ccc2f7b4637ae11aacd99841baa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6c8166abe60b491aa524c7df9c7e5cae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/c75decb9bb3b4eabac1f9a3a36d1ece7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/38bdfa206e6f4ae2b23a41616b51c0d1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/3a5015f149de4942963ad38842269f53, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/e60d50d0e6914ffbb46e819849dc15f4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/fd04739e7f4548d3b7abf54a6257f94d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601] to archive 2024-11-14T07:25:42,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:25:42,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1454cba5b9a8416ba85b45803a66b4ea 2024-11-14T07:25:42,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4271b4ff03ee463590718012702089dd 2024-11-14T07:25:42,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/35df91d6f7fa42ff83a9c8c0adef11fd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/35df91d6f7fa42ff83a9c8c0adef11fd 2024-11-14T07:25:42,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/1ad88b4f1f9b497eae6efe8fc6b12166 2024-11-14T07:25:42,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f21547976e134c79a788e822aff8bab3 2024-11-14T07:25:42,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/2f0669f27149483887970d99e01974a2 2024-11-14T07:25:42,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9a371ccc2f7b4637ae11aacd99841baa to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9a371ccc2f7b4637ae11aacd99841baa 2024-11-14T07:25:42,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/de9dcc6182424bcfba53b9c790fdf0aa 2024-11-14T07:25:42,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/039d0c7fa0874d5aa440795bd3bb9894 2024-11-14T07:25:42,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6c8166abe60b491aa524c7df9c7e5cae to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6c8166abe60b491aa524c7df9c7e5cae 2024-11-14T07:25:42,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/5e9ebf22722244f28068278f7282f196 2024-11-14T07:25:42,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6cd01df466bc48d99710ee14e67d63fb 2024-11-14T07:25:42,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/c75decb9bb3b4eabac1f9a3a36d1ece7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/c75decb9bb3b4eabac1f9a3a36d1ece7 2024-11-14T07:25:42,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/7e59e05d73124121a95726c760eed1cc 2024-11-14T07:25:42,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/6f2c83aba6364c7e804c4f5f0fa8010f 2024-11-14T07:25:42,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/38bdfa206e6f4ae2b23a41616b51c0d1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/38bdfa206e6f4ae2b23a41616b51c0d1 2024-11-14T07:25:42,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/11350fda380e43d2ad8cd49415c56046 2024-11-14T07:25:42,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/39f1097b67114e62bc5e51f4e40349ca 2024-11-14T07:25:42,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/f3f63933c08b4ef7b95006910986b754 2024-11-14T07:25:42,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/3a5015f149de4942963ad38842269f53 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/3a5015f149de4942963ad38842269f53 2024-11-14T07:25:42,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/536d50af3e594c198dd7607bcd673db8 2024-11-14T07:25:42,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/89d8cf8085d34ba2af04471158c4d02c 2024-11-14T07:25:42,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/e60d50d0e6914ffbb46e819849dc15f4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/e60d50d0e6914ffbb46e819849dc15f4 2024-11-14T07:25:42,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/dcbdbeeb78594589a7ae270c8f17a546 2024-11-14T07:25:42,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/df868d502ac24b87aed921f87f990a51 2024-11-14T07:25:42,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/d1356b7751254d64becc0c4648dc00f2 2024-11-14T07:25:42,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/fd04739e7f4548d3b7abf54a6257f94d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/fd04739e7f4548d3b7abf54a6257f94d 2024-11-14T07:25:42,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/ef00924be31d41e6bd23a036783952dc 2024-11-14T07:25:42,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/4822351e59364b569153af7877e383a0 2024-11-14T07:25:42,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/97f2da44dad84389babb59c68b18f601 2024-11-14T07:25:42,179 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits/436.seqid, newMaxSeqId=436, maxSeqId=4 2024-11-14T07:25:42,180 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. 2024-11-14T07:25:42,180 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for c7069097a7b68031cda66e228521c56e: Waiting for close lock at 1731569141203Running coprocessor pre-close hooks at 1731569141203Disabling compacts and flushes for region at 1731569141203Disabling writes for close at 1731569141203Obtaining lock to block concurrent updates at 1731569141203Preparing flush snapshotting stores in c7069097a7b68031cda66e228521c56e at 1731569141203Finished memstore snapshotting TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731569141204 (+1 ms)Flushing stores of TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e. at 1731569141204Flushing c7069097a7b68031cda66e228521c56e/A: creating writer at 1731569141204Flushing mob file c7069097a7b68031cda66e228521c56e/A: appending metadata at 1731569141210 (+6 ms)Flushing mob file c7069097a7b68031cda66e228521c56e/A: closing flushed file at 1731569141210Flushing c7069097a7b68031cda66e228521c56e/A: appending metadata at 1731569141624 (+414 ms)Flushing c7069097a7b68031cda66e228521c56e/A: closing flushed file at 1731569141624Flushing c7069097a7b68031cda66e228521c56e/B: creating writer at 1731569142038 (+414 ms)Flushing c7069097a7b68031cda66e228521c56e/B: appending metadata at 1731569142040 (+2 ms)Flushing c7069097a7b68031cda66e228521c56e/B: closing flushed file at 1731569142041 (+1 ms)Flushing c7069097a7b68031cda66e228521c56e/C: creating writer at 1731569142050 (+9 ms)Flushing c7069097a7b68031cda66e228521c56e/C: appending metadata at 1731569142053 (+3 ms)Flushing c7069097a7b68031cda66e228521c56e/C: closing flushed file at 1731569142053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17efbd44: reopening flushed file at 1731569142061 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3625ee20: reopening flushed file at 1731569142067 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47254027: reopening flushed file at 1731569142073 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c7069097a7b68031cda66e228521c56e in 877ms, sequenceid=433, compaction requested=false at 1731569142080 (+7 ms)Writing region close event to WAL at 1731569142176 (+96 ms)Running coprocessor post-close hooks at 1731569142179 (+3 ms)Closed at 1731569142180 (+1 ms) 2024-11-14T07:25:42,182 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:42,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c7069097a7b68031cda66e228521c56e, regionState=CLOSED 2024-11-14T07:25:42,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:42,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T07:25:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:42,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure c7069097a7b68031cda66e228521c56e, server=c7aa6e2790b3,46235,1731569085523 in 1.1320 sec 2024-11-14T07:25:42,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-14T07:25:42,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7069097a7b68031cda66e228521c56e, UNASSIGN in 1.1370 sec 2024-11-14T07:25:42,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T07:25:42,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1390 sec 2024-11-14T07:25:42,189 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569142188"}]},"ts":"1731569142188"} 2024-11-14T07:25:42,190 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:25:42,190 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:25:42,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-14T07:25:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T07:25:43,195 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:43,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:25:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,237 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:25:43,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,244 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits] 2024-11-14T07:25:43,251 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/03e681d45f4f4cd6908587b25887259c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/03e681d45f4f4cd6908587b25887259c 2024-11-14T07:25:43,253 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/81470b43d968412996f97532a1941ded to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/A/81470b43d968412996f97532a1941ded 2024-11-14T07:25:43,255 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/9d17c3aafc754240895ad82da7dea7ac to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/9d17c3aafc754240895ad82da7dea7ac 2024-11-14T07:25:43,257 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c049be2d76024771814e31dbc9f10ba7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/B/c049be2d76024771814e31dbc9f10ba7 2024-11-14T07:25:43,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/17de501cceb140f9a96403aeb38d10af to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/17de501cceb140f9a96403aeb38d10af 2024-11-14T07:25:43,260 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9e52b5fbfc614c75ac49fc2f97b025a9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/C/9e52b5fbfc614c75ac49fc2f97b025a9 2024-11-14T07:25:43,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits/436.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e/recovered.edits/436.seqid 2024-11-14T07:25:43,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,263 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:25:43,263 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:25:43,264 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:25:43,269 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114445623f7ecbb488683e8775e624deb30_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114445623f7ecbb488683e8775e624deb30_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,270 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444ac7e9540b84e6b88ca985c02c4a0f7_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444ac7e9540b84e6b88ca985c02c4a0f7_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,272 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144818c3639860406c945ccfe3b2a82c93_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144818c3639860406c945ccfe3b2a82c93_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,273 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bee99cc60fd4588ba43438d3e1de92c_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bee99cc60fd4588ba43438d3e1de92c_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,274 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144f788c0683954137acc221b53d45bbc7_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144f788c0683954137acc221b53d45bbc7_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,275 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114763971adfef04e11b800ecb21fe47342_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114763971adfef04e11b800ecb21fe47342_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,277 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148313f26e720c46da98b7264d99cc9d17_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148313f26e720c46da98b7264d99cc9d17_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,278 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114877e30a4e50c4605810da9b5b61e10f3_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114877e30a4e50c4605810da9b5b61e10f3_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,279 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148781056cf1604e55bfc542dd46ed8afb_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148781056cf1604e55bfc542dd46ed8afb_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,281 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148b49ffa9b2324a0fade15b07b545a6ef_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148b49ffa9b2324a0fade15b07b545a6ef_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,282 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111490812aed87bc45f99a3aa08338391d4d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111490812aed87bc45f99a3aa08338391d4d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,283 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492da87f7219a4a87a99c25d40b4aaaa9_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111492da87f7219a4a87a99c25d40b4aaaa9_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,284 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114981e061714e84f20bbf0f5631e406bf0_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114981e061714e84f20bbf0f5631e406bf0_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,285 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdec0efa39048a2992d04a0c0cd95f8_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149cdec0efa39048a2992d04a0c0cd95f8_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,287 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a1b787e97a7541c09d5d94efff79e529_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a1b787e97a7541c09d5d94efff79e529_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,289 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a7399b3c54ec4902936262c35429191f_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a7399b3c54ec4902936262c35429191f_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,290 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4bd9516d994747a486a244d5281297_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf4bd9516d994747a486a244d5281297_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,291 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e50f5d68474ee9bb59d9620ab64f9d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e50f5d68474ee9bb59d9620ab64f9d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,292 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb46496dc957470c96ef9f0a85b37b6d_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb46496dc957470c96ef9f0a85b37b6d_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,293 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb89a837402243ada65cf420a5e39183_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb89a837402243ada65cf420a5e39183_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,294 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cca413725ef14f01bee1db57dac9d566_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cca413725ef14f01bee1db57dac9d566_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,296 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ebb78747a74a4a2fb78a9729c5dc08b2_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ebb78747a74a4a2fb78a9729c5dc08b2_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,297 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f275754e2d7a4ceab52e9b03e8030fd1_c7069097a7b68031cda66e228521c56e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f275754e2d7a4ceab52e9b03e8030fd1_c7069097a7b68031cda66e228521c56e 2024-11-14T07:25:43,297 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:25:43,299 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,302 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:25:43,304 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:25:43,305 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,305 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:25:43,306 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569143305"}]},"ts":"9223372036854775807"} 2024-11-14T07:25:43,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:25:43,308 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c7069097a7b68031cda66e228521c56e, NAME => 'TestAcidGuarantees,,1731569114913.c7069097a7b68031cda66e228521c56e.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:25:43,308 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:25:43,308 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569143308"}]},"ts":"9223372036854775807"} 2024-11-14T07:25:43,310 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:25:43,311 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 116 msec 2024-11-14T07:25:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T07:25:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:25:43,348 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:43,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=245 (was 239) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1194628676_22 at /127.0.0.1:46442 [Waiting for operation #218] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_230653519_22 at /127.0.0.1:46598 [Waiting for operation #172] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1194628676_22 at /127.0.0.1:46614 [Waiting for operation #171] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_230653519_22 at /127.0.0.1:44316 [Waiting for operation #629] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=518 (was 502) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5680 (was 6148) 2024-11-14T07:25:43,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=245, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=518, ProcessCount=11, AvailableMemoryMB=5679 2024-11-14T07:25:43,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:25:43,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:25:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:25:43,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:25:43,364 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:43,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-14T07:25:43,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:25:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:43,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742102_1278 (size=959) 2024-11-14T07:25:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:43,774 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:25:43,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742103_1279 (size=53) 2024-11-14T07:25:43,896 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:25:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:44,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:25:44,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7fcaf3f9e5b875de457322a3a36b0e3, disabling compactions & flushes 2024-11-14T07:25:44,185 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. after waiting 0 ms 2024-11-14T07:25:44,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,185 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7fcaf3f9e5b875de457322a3a36b0e3: Waiting for close lock at 1731569144185Disabling compacts and flushes for region at 1731569144185Disabling writes for close at 1731569144185Writing region close event to WAL at 1731569144185Closed at 1731569144185 2024-11-14T07:25:44,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:25:44,189 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569144188"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569144188"}]},"ts":"1731569144188"} 2024-11-14T07:25:44,191 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:25:44,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:25:44,192 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569144192"}]},"ts":"1731569144192"} 2024-11-14T07:25:44,194 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:25:44,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, ASSIGN}] 2024-11-14T07:25:44,195 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, ASSIGN 2024-11-14T07:25:44,196 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:25:44,347 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f7fcaf3f9e5b875de457322a3a36b0e3, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:44,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, ASSIGN because future has completed 2024-11-14T07:25:44,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:25:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:44,511 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,511 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:25:44,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:25:44,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,515 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,517 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7fcaf3f9e5b875de457322a3a36b0e3 columnFamilyName A 2024-11-14T07:25:44,518 DEBUG [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:44,518 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(327): Store=f7fcaf3f9e5b875de457322a3a36b0e3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:44,518 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,520 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7fcaf3f9e5b875de457322a3a36b0e3 columnFamilyName B 2024-11-14T07:25:44,520 DEBUG [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:44,521 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(327): Store=f7fcaf3f9e5b875de457322a3a36b0e3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:44,521 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,523 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7fcaf3f9e5b875de457322a3a36b0e3 columnFamilyName C 2024-11-14T07:25:44,523 DEBUG [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:25:44,524 INFO [StoreOpener-f7fcaf3f9e5b875de457322a3a36b0e3-1 {}] regionserver.HStore(327): Store=f7fcaf3f9e5b875de457322a3a36b0e3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:25:44,524 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,525 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,525 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,526 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,527 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,527 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:25:44,528 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,530 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:25:44,530 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened f7fcaf3f9e5b875de457322a3a36b0e3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60043625, jitterRate=-0.10528026521205902}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:25:44,530 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:44,531 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for f7fcaf3f9e5b875de457322a3a36b0e3: Running coprocessor pre-open hook at 1731569144512Writing region info on filesystem at 1731569144512Initializing all the Stores at 1731569144514 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569144514Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569144515 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569144515Cleaning up temporary data from old regions at 1731569144527 (+12 ms)Running coprocessor post-open hooks at 1731569144531 (+4 ms)Region opened successfully at 1731569144531 2024-11-14T07:25:44,532 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., pid=67, masterSystemTime=1731569144505 2024-11-14T07:25:44,533 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,533 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:44,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f7fcaf3f9e5b875de457322a3a36b0e3, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:44,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:25:44,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T07:25:44,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 in 185 msec 2024-11-14T07:25:44,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-14T07:25:44,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, ASSIGN in 343 msec 2024-11-14T07:25:44,539 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:25:44,540 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569144539"}]},"ts":"1731569144539"} 2024-11-14T07:25:44,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:25:44,542 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:25:44,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-14T07:25:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-14T07:25:45,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:45,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,516 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e75f958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb47cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,519 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,521 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:25:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:25:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39a28136 2024-11-14T07:25:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:25:45,523 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:25:45,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,525 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134a63c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,526 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df4a0b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,527 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82c4d38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,529 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b78528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,531 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aafa0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,532 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c384e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3191bc85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b70cee2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,536 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb465b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189e6436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,539 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a4274e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,540 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ad4e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,542 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b821ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37fb21f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,545 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2cff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,546 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd80f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,547 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ffc6ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d7d757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:25:45,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:25:45,550 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:25:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:25:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:25:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cd31f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:25:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:25:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:25:45,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:25:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:25:45,552 DEBUG [Thread-1287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,552 DEBUG [Thread-1285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,552 DEBUG [Thread-1274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,552 DEBUG [Thread-1291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,552 DEBUG [Thread-1282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,553 DEBUG [Thread-1276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,553 DEBUG [Thread-1278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,553 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,554 DEBUG [Thread-1293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,554 DEBUG [Thread-1280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,555 DEBUG [Thread-1280 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,555 DEBUG [Thread-1280 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,556 DEBUG [Thread-1289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:25:45,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:25:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T07:25:45,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:25:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:45,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:45,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:45,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:25:45,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/da84ddac163a4d0eb441817dc5c03f7a is 50, key is test_row_0/A:col10/1731569145602/Put/seqid=0 2024-11-14T07:25:45,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742104_1280 (size=12001) 2024-11-14T07:25:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569205622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569205623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569205626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569205627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569205627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:45,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:45,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:45,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569205734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569205734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569205734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569205734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569205734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:45,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569205944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/da84ddac163a4d0eb441817dc5c03f7a 2024-11-14T07:25:46,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/7c106c6a4ead4c70a3ef45c9c4d97ace is 50, key is test_row_0/B:col10/1731569145602/Put/seqid=0 2024-11-14T07:25:46,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742105_1281 (size=12001) 2024-11-14T07:25:46,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569206255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/7c106c6a4ead4c70a3ef45c9c4d97ace 2024-11-14T07:25:46,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/4f4536f8d11247f482faa22586141ddb is 50, key is test_row_0/C:col10/1731569145602/Put/seqid=0 2024-11-14T07:25:46,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742106_1282 (size=12001) 2024-11-14T07:25:46,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569206764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569206764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569206764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569206764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569206764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:46,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:46,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/4f4536f8d11247f482faa22586141ddb 2024-11-14T07:25:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/da84ddac163a4d0eb441817dc5c03f7a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a 2024-11-14T07:25:46,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:25:46,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/7c106c6a4ead4c70a3ef45c9c4d97ace as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace 2024-11-14T07:25:46,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:25:46,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/4f4536f8d11247f482faa22586141ddb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb 2024-11-14T07:25:46,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:25:46,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1297ms, sequenceid=15, compaction requested=false 2024-11-14T07:25:46,900 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T07:25:46,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:46,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T07:25:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:46,940 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:25:46,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/a1cd21b30d6e477e9fdafe4eaeb5232d is 50, key is test_row_0/A:col10/1731569145626/Put/seqid=0 2024-11-14T07:25:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742107_1283 (size=12001) 2024-11-14T07:25:46,963 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/a1cd21b30d6e477e9fdafe4eaeb5232d 2024-11-14T07:25:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/be2f0b4f350d41c5a5eb6659425d1731 is 50, key is test_row_0/B:col10/1731569145626/Put/seqid=0 2024-11-14T07:25:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742108_1284 (size=12001) 2024-11-14T07:25:46,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/be2f0b4f350d41c5a5eb6659425d1731 2024-11-14T07:25:46,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/424e7ed8341243a1bf2d152f9102b905 is 50, key is test_row_0/C:col10/1731569145626/Put/seqid=0 2024-11-14T07:25:47,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742109_1285 (size=12001) 2024-11-14T07:25:47,002 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/424e7ed8341243a1bf2d152f9102b905 2024-11-14T07:25:47,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/a1cd21b30d6e477e9fdafe4eaeb5232d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d 2024-11-14T07:25:47,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:25:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/be2f0b4f350d41c5a5eb6659425d1731 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731 2024-11-14T07:25:47,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:25:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/424e7ed8341243a1bf2d152f9102b905 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905 2024-11-14T07:25:47,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:25:47,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for f7fcaf3f9e5b875de457322a3a36b0e3 in 82ms, sequenceid=38, compaction requested=false 2024-11-14T07:25:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T07:25:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T07:25:47,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T07:25:47,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-14T07:25:47,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4740 sec 2024-11-14T07:25:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T07:25:47,704 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:47,705 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T07:25:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:47,707 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:47,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:47,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:47,709 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:25:47,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:47,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:25:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:47,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/7226e3455d1341a98b1bf15f593f93ed is 50, key is test_row_0/A:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:47,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742110_1286 (size=14341) 2024-11-14T07:25:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:47,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569207817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569207820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569207821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569207822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569207825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:47,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569207935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569207935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569207935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569207935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569207935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:48,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569208144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569208145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569208145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569208145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569208145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/7226e3455d1341a98b1bf15f593f93ed 2024-11-14T07:25:48,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/0481d4675fd547f28a9139059c53abc6 is 50, key is test_row_0/B:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:48,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742111_1287 (size=12001) 2024-11-14T07:25:48,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569208455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569208455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569208455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569208455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569208455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:48,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/0481d4675fd547f28a9139059c53abc6 2024-11-14T07:25:48,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ad437fa8a7fe43979287eae29a2929bc is 50, key is test_row_0/C:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742112_1288 (size=12001) 2024-11-14T07:25:48,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ad437fa8a7fe43979287eae29a2929bc 2024-11-14T07:25:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/7226e3455d1341a98b1bf15f593f93ed as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed 2024-11-14T07:25:48,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed, entries=200, sequenceid=50, filesize=14.0 K 2024-11-14T07:25:48,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/0481d4675fd547f28a9139059c53abc6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6 2024-11-14T07:25:48,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T07:25:48,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ad437fa8a7fe43979287eae29a2929bc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc 2024-11-14T07:25:48,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T07:25:48,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7fcaf3f9e5b875de457322a3a36b0e3 in 964ms, sequenceid=50, compaction requested=true 2024-11-14T07:25:48,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:48,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:48,744 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:48,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:48,744 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:48,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:48,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:48,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:48,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:48,745 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:48,745 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:48,745 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:48,745 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,745 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:48,745 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,745 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=37.4 K 2024-11-14T07:25:48,746 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.2 K 2024-11-14T07:25:48,746 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c106c6a4ead4c70a3ef45c9c4d97ace, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569145566 2024-11-14T07:25:48,746 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting da84ddac163a4d0eb441817dc5c03f7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569145566 2024-11-14T07:25:48,746 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting be2f0b4f350d41c5a5eb6659425d1731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569145621 2024-11-14T07:25:48,747 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0481d4675fd547f28a9139059c53abc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:48,747 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1cd21b30d6e477e9fdafe4eaeb5232d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569145621 2024-11-14T07:25:48,748 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7226e3455d1341a98b1bf15f593f93ed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:48,774 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#239 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:48,774 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1e2d9b7980e44fe1a470aba66dca0f6b is 50, key is test_row_0/B:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:48,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T07:25:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:25:48,787 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:48,787 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/bd4a0179cf6b4011b218c46fb69eae55 is 50, key is test_row_0/A:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:48,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742113_1289 (size=12104) 2024-11-14T07:25:48,802 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1e2d9b7980e44fe1a470aba66dca0f6b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1e2d9b7980e44fe1a470aba66dca0f6b 2024-11-14T07:25:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0144ef0dcd904f24a99e27a95e973a59 is 50, key is test_row_0/A:col10/1731569147821/Put/seqid=0 2024-11-14T07:25:48,812 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 1e2d9b7980e44fe1a470aba66dca0f6b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:48,812 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:48,812 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569148744; duration=0sec 2024-11-14T07:25:48,813 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:48,813 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:48,813 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:48,816 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:48,816 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:48,816 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:48,816 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.2 K 2024-11-14T07:25:48,816 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4536f8d11247f482faa22586141ddb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569145566 2024-11-14T07:25:48,817 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 424e7ed8341243a1bf2d152f9102b905, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569145621 2024-11-14T07:25:48,817 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ad437fa8a7fe43979287eae29a2929bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:48,835 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:48,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/d84a36bb20884e6abcfe85f5447f4420 is 50, key is test_row_0/C:col10/1731569147775/Put/seqid=0 2024-11-14T07:25:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:48,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742114_1290 (size=12104) 2024-11-14T07:25:48,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742115_1291 (size=12001) 2024-11-14T07:25:48,878 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0144ef0dcd904f24a99e27a95e973a59 2024-11-14T07:25:48,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bead5cb30a154359916fb16c110cd8c0 is 50, key is test_row_0/B:col10/1731569147821/Put/seqid=0 2024-11-14T07:25:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742116_1292 (size=12104) 2024-11-14T07:25:48,913 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/d84a36bb20884e6abcfe85f5447f4420 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d84a36bb20884e6abcfe85f5447f4420 2024-11-14T07:25:48,922 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into d84a36bb20884e6abcfe85f5447f4420(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:48,922 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:48,922 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569148745; duration=0sec 2024-11-14T07:25:48,923 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:48,923 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:48,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742117_1293 (size=12001) 2024-11-14T07:25:48,924 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bead5cb30a154359916fb16c110cd8c0 2024-11-14T07:25:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f1b4af233c4c4fd5837e504ee4261a5f is 50, key is test_row_0/C:col10/1731569147821/Put/seqid=0 2024-11-14T07:25:48,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742118_1294 (size=12001) 2024-11-14T07:25:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:48,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569208979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569208980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569208980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569208982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569208982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569209094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569209094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569209095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569209095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569209095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,276 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/bd4a0179cf6b4011b218c46fb69eae55 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/bd4a0179cf6b4011b218c46fb69eae55 2024-11-14T07:25:49,280 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into bd4a0179cf6b4011b218c46fb69eae55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:49,280 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:49,280 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569148744; duration=0sec 2024-11-14T07:25:49,281 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:49,281 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569209304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569209305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569209305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569209305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569209305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f1b4af233c4c4fd5837e504ee4261a5f 2024-11-14T07:25:49,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0144ef0dcd904f24a99e27a95e973a59 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59 2024-11-14T07:25:49,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:25:49,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bead5cb30a154359916fb16c110cd8c0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0 2024-11-14T07:25:49,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:25:49,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f1b4af233c4c4fd5837e504ee4261a5f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f 2024-11-14T07:25:49,369 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:25:49,370 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f7fcaf3f9e5b875de457322a3a36b0e3 in 588ms, sequenceid=74, compaction requested=false 2024-11-14T07:25:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T07:25:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T07:25:49,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T07:25:49,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-11-14T07:25:49,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6680 sec 2024-11-14T07:25:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:49,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:49,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:25:49,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/3bbb7b1db7ab4ebb9c6b915701163fa7 is 50, key is test_row_0/A:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:49,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742119_1295 (size=12001) 2024-11-14T07:25:49,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569209633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569209634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569209635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569209635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569209636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569209745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569209745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569209745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569209745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569209745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T07:25:49,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:49,855 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T07:25:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:25:49,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:49,857 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:49,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569209955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569209955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569209956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569209957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569209957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:25:50,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T07:25:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:50,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/3bbb7b1db7ab4ebb9c6b915701163fa7 2024-11-14T07:25:50,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/708afb7d40a04df99ec44776cd84491e is 50, key is test_row_0/B:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:50,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742120_1296 (size=12001) 2024-11-14T07:25:50,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/708afb7d40a04df99ec44776cd84491e 2024-11-14T07:25:50,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b5fc626188c74f4db0825a83b22b62fe is 50, key is test_row_0/C:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742121_1297 (size=12001) 2024-11-14T07:25:50,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b5fc626188c74f4db0825a83b22b62fe 2024-11-14T07:25:50,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/3bbb7b1db7ab4ebb9c6b915701163fa7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7 2024-11-14T07:25:50,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T07:25:50,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/708afb7d40a04df99ec44776cd84491e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e 2024-11-14T07:25:50,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T07:25:50,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b5fc626188c74f4db0825a83b22b62fe as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe 2024-11-14T07:25:50,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T07:25:50,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f7fcaf3f9e5b875de457322a3a36b0e3 in 484ms, sequenceid=92, compaction requested=true 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:50,101 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:50,101 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:50,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:50,102 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:50,102 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:50,102 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,102 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/bd4a0179cf6b4011b218c46fb69eae55, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.3 K 2024-11-14T07:25:50,103 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4a0179cf6b4011b218c46fb69eae55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:50,103 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:50,103 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:50,103 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,103 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1e2d9b7980e44fe1a470aba66dca0f6b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.3 K 2024-11-14T07:25:50,103 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0144ef0dcd904f24a99e27a95e973a59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569147820 2024-11-14T07:25:50,103 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbb7b1db7ab4ebb9c6b915701163fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:50,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2d9b7980e44fe1a470aba66dca0f6b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:50,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting bead5cb30a154359916fb16c110cd8c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569147820 2024-11-14T07:25:50,104 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 708afb7d40a04df99ec44776cd84491e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:50,115 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:50,116 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/dad5b966f0fd44ad9d7a01e7276fad47 is 50, key is test_row_0/A:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:50,123 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:50,124 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/3280d5688ccc4c608bc845cd6fa0ec7e is 50, key is test_row_0/B:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:50,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742122_1298 (size=12207) 2024-11-14T07:25:50,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742123_1299 (size=12207) 2024-11-14T07:25:50,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T07:25:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T07:25:50,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e2fc7841cb844ba599dae28cc2756576 is 50, key is test_row_0/A:col10/1731569149633/Put/seqid=0 2024-11-14T07:25:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742124_1300 (size=12001) 2024-11-14T07:25:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:25:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:50,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569210277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569210278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569210278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569210281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569210282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569210384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569210384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569210384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569210395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569210395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:25:50,541 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/3280d5688ccc4c608bc845cd6fa0ec7e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/3280d5688ccc4c608bc845cd6fa0ec7e 2024-11-14T07:25:50,551 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 3280d5688ccc4c608bc845cd6fa0ec7e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:50,551 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:50,551 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569150101; duration=0sec 2024-11-14T07:25:50,551 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:50,552 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:50,552 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:50,554 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:50,554 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:50,554 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,554 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d84a36bb20884e6abcfe85f5447f4420, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.3 K 2024-11-14T07:25:50,555 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d84a36bb20884e6abcfe85f5447f4420, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731569147775 2024-11-14T07:25:50,555 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b4af233c4c4fd5837e504ee4261a5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569147820 2024-11-14T07:25:50,555 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b5fc626188c74f4db0825a83b22b62fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:50,565 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:50,566 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/52e78fed6c674e0aaffdda2a47e22539 is 50, key is test_row_0/C:col10/1731569149615/Put/seqid=0 2024-11-14T07:25:50,572 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/dad5b966f0fd44ad9d7a01e7276fad47 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/dad5b966f0fd44ad9d7a01e7276fad47 2024-11-14T07:25:50,580 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into dad5b966f0fd44ad9d7a01e7276fad47(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:50,580 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:50,580 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569150101; duration=0sec 2024-11-14T07:25:50,580 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:50,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e2fc7841cb844ba599dae28cc2756576 2024-11-14T07:25:50,582 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:50,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742125_1301 (size=12207) 2024-11-14T07:25:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569210595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,599 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/52e78fed6c674e0aaffdda2a47e22539 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/52e78fed6c674e0aaffdda2a47e22539 2024-11-14T07:25:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569210595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569210595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569210605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569210605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/41657b8c88d04c06b793f6e45b9153b1 is 50, key is test_row_0/B:col10/1731569149633/Put/seqid=0 2024-11-14T07:25:50,609 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into 52e78fed6c674e0aaffdda2a47e22539(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:50,609 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:50,609 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569150101; duration=0sec 2024-11-14T07:25:50,609 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:50,609 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:50,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742126_1302 (size=12001) 2024-11-14T07:25:50,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/41657b8c88d04c06b793f6e45b9153b1 2024-11-14T07:25:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/caa3720547a1402f8cd098837343f3f0 is 50, key is test_row_0/C:col10/1731569149633/Put/seqid=0 2024-11-14T07:25:50,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742127_1303 (size=12001) 2024-11-14T07:25:50,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/caa3720547a1402f8cd098837343f3f0 2024-11-14T07:25:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e2fc7841cb844ba599dae28cc2756576 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576 2024-11-14T07:25:50,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T07:25:50,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/41657b8c88d04c06b793f6e45b9153b1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1 2024-11-14T07:25:50,714 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T07:25:50,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/caa3720547a1402f8cd098837343f3f0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0 2024-11-14T07:25:50,727 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T07:25:50,728 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f7fcaf3f9e5b875de457322a3a36b0e3 in 564ms, sequenceid=113, compaction requested=false 2024-11-14T07:25:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T07:25:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T07:25:50,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T07:25:50,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 873 msec 2024-11-14T07:25:50,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 881 msec 2024-11-14T07:25:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:50,908 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:50,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T07:25:50,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9fb8294804494cffbde5a333341c3ad5 is 50, key is test_row_0/A:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:50,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742128_1304 (size=16831) 2024-11-14T07:25:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569210937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569210941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569210942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569210942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569210945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T07:25:50,995 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:50,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T07:25:50,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:50,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:50,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569211055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569211055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569211055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569211055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569211055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:51,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569211265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569211265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569211265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569211265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569211265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:51,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9fb8294804494cffbde5a333341c3ad5 2024-11-14T07:25:51,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/8cbe5612572348d4ad44a795d3ce3e2c is 50, key is test_row_0/B:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:51,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742129_1305 (size=12101) 2024-11-14T07:25:51,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/8cbe5612572348d4ad44a795d3ce3e2c 2024-11-14T07:25:51,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/6ceab01002e741f8b3ce8b06e9d6da71 is 50, key is test_row_0/C:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:51,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742130_1306 (size=12101) 2024-11-14T07:25:51,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569211574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569211574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569211574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569211574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569211574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:51,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:51,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:51,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/6ceab01002e741f8b3ce8b06e9d6da71 2024-11-14T07:25:51,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9fb8294804494cffbde5a333341c3ad5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5 2024-11-14T07:25:51,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5, entries=250, sequenceid=134, filesize=16.4 K 2024-11-14T07:25:51,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/8cbe5612572348d4ad44a795d3ce3e2c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c 2024-11-14T07:25:51,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T07:25:51,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/6ceab01002e741f8b3ce8b06e9d6da71 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71 2024-11-14T07:25:51,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T07:25:51,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for f7fcaf3f9e5b875de457322a3a36b0e3 in 897ms, sequenceid=134, compaction requested=true 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:51,805 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:51,805 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:51,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:51,806 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:51,806 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:51,806 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:51,806 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:51,806 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,806 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,806 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/dad5b966f0fd44ad9d7a01e7276fad47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=40.1 K 2024-11-14T07:25:51,806 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/3280d5688ccc4c608bc845cd6fa0ec7e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.5 K 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad5b966f0fd44ad9d7a01e7276fad47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3280d5688ccc4c608bc845cd6fa0ec7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2fc7841cb844ba599dae28cc2756576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731569149632 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 41657b8c88d04c06b793f6e45b9153b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731569149632 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb8294804494cffbde5a333341c3ad5, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:51,807 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cbe5612572348d4ad44a795d3ce3e2c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:51,817 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:51,817 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:51,818 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/872f3bf4d9604620ad06534b1c3159cd is 50, key is test_row_0/A:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:51,818 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/5968db5713554267ab7f4c9b4c6041c5 is 50, key is test_row_0/B:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:51,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742132_1308 (size=12409) 2024-11-14T07:25:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742131_1307 (size=12409) 2024-11-14T07:25:51,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/872f3bf4d9604620ad06534b1c3159cd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/872f3bf4d9604620ad06534b1c3159cd 2024-11-14T07:25:51,838 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into 872f3bf4d9604620ad06534b1c3159cd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:51,838 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:51,838 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569151805; duration=0sec 2024-11-14T07:25:51,838 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:51,838 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:51,838 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:51,840 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:51,841 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:51,841 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,841 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/52e78fed6c674e0aaffdda2a47e22539, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.5 K 2024-11-14T07:25:51,841 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e78fed6c674e0aaffdda2a47e22539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731569148979 2024-11-14T07:25:51,841 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa3720547a1402f8cd098837343f3f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731569149632 2024-11-14T07:25:51,841 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ceab01002e741f8b3ce8b06e9d6da71, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:51,849 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:51,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/da016d29298248f69de79500739c2795 is 50, key is test_row_0/C:col10/1731569150907/Put/seqid=0 2024-11-14T07:25:51,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742133_1309 (size=12409) 2024-11-14T07:25:51,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T07:25:51,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:51,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T07:25:51,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/2cf77571afa041e7b7c71cc23c126b86 is 50, key is test_row_0/A:col10/1731569150940/Put/seqid=0 2024-11-14T07:25:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742134_1310 (size=12151) 2024-11-14T07:25:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:52,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569212099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569212100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569212100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569212101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569212103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:52,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569212205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569212215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569212215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569212215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569212215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,228 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/5968db5713554267ab7f4c9b4c6041c5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5968db5713554267ab7f4c9b4c6041c5 2024-11-14T07:25:52,232 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 5968db5713554267ab7f4c9b4c6041c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:52,232 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:52,232 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569151805; duration=0sec 2024-11-14T07:25:52,232 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:52,232 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:52,273 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/da016d29298248f69de79500739c2795 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/da016d29298248f69de79500739c2795 2024-11-14T07:25:52,277 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into da016d29298248f69de79500739c2795(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:52,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:52,277 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569151805; duration=0sec 2024-11-14T07:25:52,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:52,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:52,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/2cf77571afa041e7b7c71cc23c126b86 2024-11-14T07:25:52,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1913b2a8c46143f6b200d8ff32f0eb29 is 50, key is test_row_0/B:col10/1731569150940/Put/seqid=0 2024-11-14T07:25:52,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742135_1311 (size=12151) 2024-11-14T07:25:52,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569212415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569212424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569212424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569212424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569212424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569212725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569212735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569212735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569212735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569212735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:52,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1913b2a8c46143f6b200d8ff32f0eb29 2024-11-14T07:25:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/15c222d59b904568981be6c087ab55ba is 50, key is test_row_0/C:col10/1731569150940/Put/seqid=0 2024-11-14T07:25:52,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742136_1312 (size=12151) 2024-11-14T07:25:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:53,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/15c222d59b904568981be6c087ab55ba 2024-11-14T07:25:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/2cf77571afa041e7b7c71cc23c126b86 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86 2024-11-14T07:25:53,207 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T07:25:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-14T07:25:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1913b2a8c46143f6b200d8ff32f0eb29 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29 2024-11-14T07:25:53,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T07:25:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/15c222d59b904568981be6c087ab55ba as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba 2024-11-14T07:25:53,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T07:25:53,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1299ms, sequenceid=155, compaction requested=false 2024-11-14T07:25:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T07:25:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T07:25:53,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T07:25:53,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2240 sec 2024-11-14T07:25:53,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2300 sec 2024-11-14T07:25:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:53,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:53,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T07:25:53,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/14a4746aa7fa4bd8952d2e1df2e459f6 is 50, key is test_row_0/A:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569213258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569213259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569213259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569213260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569213261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742137_1313 (size=12151) 2024-11-14T07:25:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569213374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569213374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569213375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569213375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569213375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569213585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569213585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569213585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569213585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569213585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/14a4746aa7fa4bd8952d2e1df2e459f6 2024-11-14T07:25:53,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/65e24a43edcd46aeb07f8e9b9d0fc031 is 50, key is test_row_0/B:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:53,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742138_1314 (size=12151) 2024-11-14T07:25:53,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569213895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569213895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569213895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569213896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569213896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:53,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/65e24a43edcd46aeb07f8e9b9d0fc031 2024-11-14T07:25:54,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/530bdeffb6f643babe1658f5c9cc9747 is 50, key is test_row_0/C:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:54,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742139_1315 (size=12151) 2024-11-14T07:25:54,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/530bdeffb6f643babe1658f5c9cc9747 2024-11-14T07:25:54,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/14a4746aa7fa4bd8952d2e1df2e459f6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6 2024-11-14T07:25:54,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6, entries=150, sequenceid=176, filesize=11.9 K 2024-11-14T07:25:54,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/65e24a43edcd46aeb07f8e9b9d0fc031 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031 2024-11-14T07:25:54,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031, entries=150, sequenceid=176, filesize=11.9 K 2024-11-14T07:25:54,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/530bdeffb6f643babe1658f5c9cc9747 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747 2024-11-14T07:25:54,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747, entries=150, sequenceid=176, filesize=11.9 K 2024-11-14T07:25:54,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f7fcaf3f9e5b875de457322a3a36b0e3 in 998ms, sequenceid=176, compaction requested=true 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:54,235 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:54,235 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:54,236 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:54,236 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:54,236 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:54,237 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/872f3bf4d9604620ad06534b1c3159cd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.9 K 2024-11-14T07:25:54,237 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:54,237 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:54,237 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:54,237 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5968db5713554267ab7f4c9b4c6041c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.9 K 2024-11-14T07:25:54,238 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872f3bf4d9604620ad06534b1c3159cd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:54,238 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5968db5713554267ab7f4c9b4c6041c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:54,238 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf77571afa041e7b7c71cc23c126b86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731569150931 2024-11-14T07:25:54,238 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1913b2a8c46143f6b200d8ff32f0eb29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731569150931 2024-11-14T07:25:54,239 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a4746aa7fa4bd8952d2e1df2e459f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:54,239 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e24a43edcd46aeb07f8e9b9d0fc031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:54,250 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:54,251 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/6d1463b2a9aa407289eb2480a7fc8dde is 50, key is test_row_0/B:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:54,266 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:54,267 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/eeba4529b2b64160a8accddf1106fd81 is 50, key is test_row_0/A:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742140_1316 (size=12561) 2024-11-14T07:25:54,284 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/6d1463b2a9aa407289eb2480a7fc8dde as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/6d1463b2a9aa407289eb2480a7fc8dde 2024-11-14T07:25:54,290 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 6d1463b2a9aa407289eb2480a7fc8dde(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:54,290 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:54,290 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569154235; duration=0sec 2024-11-14T07:25:54,291 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:54,291 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:54,291 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:54,292 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:54,292 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:54,292 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:54,292 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/da016d29298248f69de79500739c2795, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=35.9 K 2024-11-14T07:25:54,293 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting da016d29298248f69de79500739c2795, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731569150275 2024-11-14T07:25:54,293 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c222d59b904568981be6c087ab55ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731569150931 2024-11-14T07:25:54,294 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 530bdeffb6f643babe1658f5c9cc9747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:54,313 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:54,313 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ad5ec1f54aea45bdb6e346d5ac53bc64 is 50, key is test_row_0/C:col10/1731569152099/Put/seqid=0 2024-11-14T07:25:54,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742141_1317 (size=12561) 2024-11-14T07:25:54,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742142_1318 (size=12561) 2024-11-14T07:25:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:54,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:54,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c09a5e6aa59e49e0a32d234088d88f65 is 50, key is test_row_0/A:col10/1731569154409/Put/seqid=0 2024-11-14T07:25:54,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742143_1319 (size=12151) 2024-11-14T07:25:54,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c09a5e6aa59e49e0a32d234088d88f65 2024-11-14T07:25:54,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569214440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569214446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569214446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569214448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569214449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/ca69d34e3fb14338adcfe1b87c4f52b0 is 50, key is test_row_0/B:col10/1731569154409/Put/seqid=0 2024-11-14T07:25:54,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742144_1320 (size=12151) 2024-11-14T07:25:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569214554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569214565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569214565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569214565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569214566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,731 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/eeba4529b2b64160a8accddf1106fd81 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/eeba4529b2b64160a8accddf1106fd81 2024-11-14T07:25:54,738 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into eeba4529b2b64160a8accddf1106fd81(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:54,738 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:54,738 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569154235; duration=0sec 2024-11-14T07:25:54,738 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:54,738 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:54,748 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ad5ec1f54aea45bdb6e346d5ac53bc64 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad5ec1f54aea45bdb6e346d5ac53bc64 2024-11-14T07:25:54,754 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into ad5ec1f54aea45bdb6e346d5ac53bc64(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:54,755 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:54,755 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569154235; duration=0sec 2024-11-14T07:25:54,755 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:54,755 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:54,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569214765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569214775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569214775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569214775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569214776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:54,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/ca69d34e3fb14338adcfe1b87c4f52b0 2024-11-14T07:25:54,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/d342a65d498a460f9903559e41109dd3 is 50, key is test_row_0/C:col10/1731569154409/Put/seqid=0 2024-11-14T07:25:54,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742145_1321 (size=12151) 2024-11-14T07:25:54,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/d342a65d498a460f9903559e41109dd3 2024-11-14T07:25:54,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c09a5e6aa59e49e0a32d234088d88f65 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65 2024-11-14T07:25:55,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T07:25:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/ca69d34e3fb14338adcfe1b87c4f52b0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0 2024-11-14T07:25:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T07:25:55,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/d342a65d498a460f9903559e41109dd3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3 2024-11-14T07:25:55,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T07:25:55,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f7fcaf3f9e5b875de457322a3a36b0e3 in 610ms, sequenceid=196, compaction requested=false 2024-11-14T07:25:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:55,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:55,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:55,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/90f3dc359fb3476ea9f1b553d11f77b6 is 50, key is test_row_0/A:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:55,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569215107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569215107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569215109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569215110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569215115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742146_1322 (size=12151) 2024-11-14T07:25:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T07:25:55,164 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:55,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T07:25:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:55,166 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:55,167 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:55,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569215225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569215225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569215225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569215225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569215230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:55,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:55,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:55,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569215434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569215435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569215435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569215435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569215446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:55,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/90f3dc359fb3476ea9f1b553d11f77b6 2024-11-14T07:25:55,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/e12587f108d14594bed5e3d201215e70 is 50, key is test_row_0/B:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:55,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742147_1323 (size=12151) 2024-11-14T07:25:55,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/e12587f108d14594bed5e3d201215e70 2024-11-14T07:25:55,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/a9a1666ec39144118a462c322855bc0a is 50, key is test_row_0/C:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:55,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742148_1324 (size=12151) 2024-11-14T07:25:55,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:55,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569215745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569215745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569215745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569215752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569215754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:55,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:55,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:55,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:55,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:55,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/a9a1666ec39144118a462c322855bc0a 2024-11-14T07:25:55,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/90f3dc359fb3476ea9f1b553d11f77b6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6 2024-11-14T07:25:55,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T07:25:55,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/e12587f108d14594bed5e3d201215e70 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70 2024-11-14T07:25:55,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T07:25:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/a9a1666ec39144118a462c322855bc0a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a 2024-11-14T07:25:55,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T07:25:55,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f7fcaf3f9e5b875de457322a3a36b0e3 in 901ms, sequenceid=216, compaction requested=true 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:55,978 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:55,978 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:55,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:55,979 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,979 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:55,979 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/eeba4529b2b64160a8accddf1106fd81, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.0 K 2024-11-14T07:25:55,979 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/6d1463b2a9aa407289eb2480a7fc8dde, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.0 K 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeba4529b2b64160a8accddf1106fd81, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:55,979 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d1463b2a9aa407289eb2480a7fc8dde, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:55,980 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09a5e6aa59e49e0a32d234088d88f65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731569153259 2024-11-14T07:25:55,980 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ca69d34e3fb14338adcfe1b87c4f52b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731569153259 2024-11-14T07:25:55,980 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f3dc359fb3476ea9f1b553d11f77b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:55,980 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e12587f108d14594bed5e3d201215e70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:55,986 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:55,987 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fcc48e340e3448c0a9817ef79787526a is 50, key is test_row_0/B:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:55,987 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:55,987 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e6740e73acb5426591102ca0fbc7d86c is 50, key is test_row_0/A:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742150_1326 (size=12663) 2024-11-14T07:25:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742149_1325 (size=12663) 2024-11-14T07:25:56,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T07:25:56,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,086 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T07:25:56,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e33c53e93ae942f58ea91ead9b8538e8 is 50, key is test_row_0/A:col10/1731569155106/Put/seqid=0 2024-11-14T07:25:56,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742151_1327 (size=12151) 2024-11-14T07:25:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:56,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:56,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569216273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569216274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569216276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569216276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569216277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:56,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569216384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569216384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569216385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569216386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569216386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,405 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e6740e73acb5426591102ca0fbc7d86c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e6740e73acb5426591102ca0fbc7d86c 2024-11-14T07:25:56,412 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into e6740e73acb5426591102ca0fbc7d86c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,412 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,412 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569155978; duration=0sec 2024-11-14T07:25:56,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:56,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:56,413 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:56,415 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fcc48e340e3448c0a9817ef79787526a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcc48e340e3448c0a9817ef79787526a 2024-11-14T07:25:56,416 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:56,416 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:56,416 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,416 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad5ec1f54aea45bdb6e346d5ac53bc64, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.0 K 2024-11-14T07:25:56,417 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5ec1f54aea45bdb6e346d5ac53bc64, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731569152098 2024-11-14T07:25:56,417 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d342a65d498a460f9903559e41109dd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731569153259 2024-11-14T07:25:56,418 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a1666ec39144118a462c322855bc0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:56,423 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into fcc48e340e3448c0a9817ef79787526a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,423 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,423 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569155978; duration=0sec 2024-11-14T07:25:56,423 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,423 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:56,430 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:56,431 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b106ddf5c98c40ffb6e60a356b70bbd3 is 50, key is test_row_0/C:col10/1731569154445/Put/seqid=0 2024-11-14T07:25:56,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742152_1328 (size=12663) 2024-11-14T07:25:56,451 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b106ddf5c98c40ffb6e60a356b70bbd3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b106ddf5c98c40ffb6e60a356b70bbd3 2024-11-14T07:25:56,457 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into b106ddf5c98c40ffb6e60a356b70bbd3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,457 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,457 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569155978; duration=0sec 2024-11-14T07:25:56,457 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,457 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:56,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e33c53e93ae942f58ea91ead9b8538e8 2024-11-14T07:25:56,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/a21ffc75cf3c4b56a86c64ca43a77748 is 50, key is test_row_0/B:col10/1731569155106/Put/seqid=0 2024-11-14T07:25:56,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742153_1329 (size=12151) 2024-11-14T07:25:56,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/a21ffc75cf3c4b56a86c64ca43a77748 2024-11-14T07:25:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/e417ebbe351b475e9403f414c3783106 is 50, key is test_row_0/C:col10/1731569155106/Put/seqid=0 2024-11-14T07:25:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742154_1330 (size=12151) 2024-11-14T07:25:56,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/e417ebbe351b475e9403f414c3783106 2024-11-14T07:25:56,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/e33c53e93ae942f58ea91ead9b8538e8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8 2024-11-14T07:25:56,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:25:56,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/a21ffc75cf3c4b56a86c64ca43a77748 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748 2024-11-14T07:25:56,581 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:25:56,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/e417ebbe351b475e9403f414c3783106 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106 2024-11-14T07:25:56,589 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:25:56,591 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for f7fcaf3f9e5b875de457322a3a36b0e3 in 506ms, sequenceid=234, compaction requested=false 2024-11-14T07:25:56,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T07:25:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T07:25:56,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T07:25:56,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-14T07:25:56,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4310 sec 2024-11-14T07:25:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:56,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:56,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T07:25:56,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/b1eb1f5e5e89427dad3dfc8e256d070f is 50, key is test_row_0/A:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569216614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569216615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569216615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569216616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569216616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742155_1331 (size=12251) 2024-11-14T07:25:56,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/b1eb1f5e5e89427dad3dfc8e256d070f 2024-11-14T07:25:56,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/5fb11acb9392422d98602cf8597dbdf1 is 50, key is test_row_0/B:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742156_1332 (size=12251) 2024-11-14T07:25:56,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/5fb11acb9392422d98602cf8597dbdf1 2024-11-14T07:25:56,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/dcb6a83848a54ae7b63e214520c2f6ae is 50, key is test_row_0/C:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742157_1333 (size=12251) 2024-11-14T07:25:56,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/dcb6a83848a54ae7b63e214520c2f6ae 2024-11-14T07:25:56,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/b1eb1f5e5e89427dad3dfc8e256d070f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f 2024-11-14T07:25:56,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T07:25:56,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/5fb11acb9392422d98602cf8597dbdf1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1 2024-11-14T07:25:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569216724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569216724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569216725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569216725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569216725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T07:25:56,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/dcb6a83848a54ae7b63e214520c2f6ae as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae 2024-11-14T07:25:56,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T07:25:56,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for f7fcaf3f9e5b875de457322a3a36b0e3 in 141ms, sequenceid=258, compaction requested=true 2024-11-14T07:25:56,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,740 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,740 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:56,741 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:56,741 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:56,741 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,741 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e6740e73acb5426591102ca0fbc7d86c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.2 K 2024-11-14T07:25:56,741 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:56,741 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:56,742 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,742 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcc48e340e3448c0a9817ef79787526a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.2 K 2024-11-14T07:25:56,742 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6740e73acb5426591102ca0fbc7d86c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:56,742 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc48e340e3448c0a9817ef79787526a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:56,742 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a21ffc75cf3c4b56a86c64ca43a77748, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569155106 2024-11-14T07:25:56,742 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33c53e93ae942f58ea91ead9b8538e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569155106 2024-11-14T07:25:56,743 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb11acb9392422d98602cf8597dbdf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:56,743 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1eb1f5e5e89427dad3dfc8e256d070f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:56,751 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:56,752 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/7083b770e9b849278a5ba2de1a0bf22d is 50, key is test_row_0/B:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,770 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:56,771 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/2d0a19e3bb244c279fe481924384d87c is 50, key is test_row_0/A:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742158_1334 (size=12865) 2024-11-14T07:25:56,799 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/7083b770e9b849278a5ba2de1a0bf22d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7083b770e9b849278a5ba2de1a0bf22d 2024-11-14T07:25:56,804 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 7083b770e9b849278a5ba2de1a0bf22d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,804 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,805 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569156740; duration=0sec 2024-11-14T07:25:56,805 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:56,805 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:56,805 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:56,806 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:56,806 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:56,806 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:56,806 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b106ddf5c98c40ffb6e60a356b70bbd3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.2 K 2024-11-14T07:25:56,807 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b106ddf5c98c40ffb6e60a356b70bbd3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731569154443 2024-11-14T07:25:56,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742159_1335 (size=12865) 2024-11-14T07:25:56,807 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e417ebbe351b475e9403f414c3783106, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569155106 2024-11-14T07:25:56,808 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb6a83848a54ae7b63e214520c2f6ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:56,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/2d0a19e3bb244c279fe481924384d87c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2d0a19e3bb244c279fe481924384d87c 2024-11-14T07:25:56,826 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:56,827 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/dc418dabbe2c4fa78f7486de10821c7c is 50, key is test_row_0/C:col10/1731569156596/Put/seqid=0 2024-11-14T07:25:56,832 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into 2d0a19e3bb244c279fe481924384d87c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,832 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569156739; duration=0sec 2024-11-14T07:25:56,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:56,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742160_1336 (size=12865) 2024-11-14T07:25:56,875 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/dc418dabbe2c4fa78f7486de10821c7c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dc418dabbe2c4fa78f7486de10821c7c 2024-11-14T07:25:56,888 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into dc418dabbe2c4fa78f7486de10821c7c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:56,888 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:56,888 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569156740; duration=0sec 2024-11-14T07:25:56,888 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:56,888 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:56,939 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:56,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T07:25:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:56,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c9780be57f4142dc88f25fab3ff0cd1b is 50, key is test_row_0/A:col10/1731569156937/Put/seqid=0 2024-11-14T07:25:56,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742161_1337 (size=14741) 2024-11-14T07:25:56,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c9780be57f4142dc88f25fab3ff0cd1b 2024-11-14T07:25:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569216958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569216960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569216961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569216966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569216966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:56,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/caed516d3c8c4ec6b3d8b13848afb48a is 50, key is test_row_0/B:col10/1731569156937/Put/seqid=0 2024-11-14T07:25:57,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742162_1338 (size=12301) 2024-11-14T07:25:57,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/caed516d3c8c4ec6b3d8b13848afb48a 2024-11-14T07:25:57,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/41798215fb3b425da2bdf157fded1255 is 50, key is test_row_0/C:col10/1731569156937/Put/seqid=0 2024-11-14T07:25:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742163_1339 (size=12301) 2024-11-14T07:25:57,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/41798215fb3b425da2bdf157fded1255 2024-11-14T07:25:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c9780be57f4142dc88f25fab3ff0cd1b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b 2024-11-14T07:25:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569217075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569217075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569217075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569217076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569217076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b, entries=200, sequenceid=279, filesize=14.4 K 2024-11-14T07:25:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/caed516d3c8c4ec6b3d8b13848afb48a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a 2024-11-14T07:25:57,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T07:25:57,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/41798215fb3b425da2bdf157fded1255 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255 2024-11-14T07:25:57,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255, entries=150, sequenceid=279, filesize=12.0 K 2024-11-14T07:25:57,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for f7fcaf3f9e5b875de457322a3a36b0e3 in 157ms, sequenceid=279, compaction requested=false 2024-11-14T07:25:57,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:57,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:57,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T07:25:57,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/64c121b20ecf4e678f176f56c22fcb9a is 50, key is test_row_0/A:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742164_1340 (size=14741) 2024-11-14T07:25:57,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/64c121b20ecf4e678f176f56c22fcb9a 2024-11-14T07:25:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569217294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569217298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569217298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569217299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569217300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/34179b4cd8904ed9b34007bd8438b74c is 50, key is test_row_0/B:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742165_1341 (size=12301) 2024-11-14T07:25:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T07:25:57,315 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:57,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:57,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/34179b4cd8904ed9b34007bd8438b74c 2024-11-14T07:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T07:25:57,319 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:57,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:57,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:25:57,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9e962439f7f8481eae2ddb4aff937750 is 50, key is test_row_0/C:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742166_1342 (size=12301) 2024-11-14T07:25:57,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569217405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569217414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569217414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569217414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569217414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:25:57,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:25:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569217615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:25:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569217624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569217624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569217624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569217624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:25:57,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9e962439f7f8481eae2ddb4aff937750 2024-11-14T07:25:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/64c121b20ecf4e678f176f56c22fcb9a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a 2024-11-14T07:25:57,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a, entries=200, sequenceid=301, filesize=14.4 K 2024-11-14T07:25:57,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/34179b4cd8904ed9b34007bd8438b74c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c 2024-11-14T07:25:57,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T07:25:57,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9e962439f7f8481eae2ddb4aff937750 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750 2024-11-14T07:25:57,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T07:25:57,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f7fcaf3f9e5b875de457322a3a36b0e3 in 478ms, sequenceid=301, compaction requested=true 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:57,764 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:57,764 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:57,765 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:57,765 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:57,765 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,765 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2d0a19e3bb244c279fe481924384d87c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=41.4 K 2024-11-14T07:25:57,766 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:57,766 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:57,766 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,766 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7083b770e9b849278a5ba2de1a0bf22d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.6 K 2024-11-14T07:25:57,766 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d0a19e3bb244c279fe481924384d87c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:57,766 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7083b770e9b849278a5ba2de1a0bf22d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:57,767 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting caed516d3c8c4ec6b3d8b13848afb48a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731569156604 2024-11-14T07:25:57,767 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9780be57f4142dc88f25fab3ff0cd1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731569156604 2024-11-14T07:25:57,767 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 34179b4cd8904ed9b34007bd8438b74c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:57,767 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c121b20ecf4e678f176f56c22fcb9a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:57,775 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:57,775 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c0eb736613c342c88c06834901361919 is 50, key is test_row_0/B:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,776 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:57,776 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/8c1c24ecabd0476f969da9512a9aaf5f is 50, key is test_row_0/A:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T07:25:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:25:57,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742167_1343 (size=13017) 2024-11-14T07:25:57,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/22a7d2bbcd704f4b80c487f864c84cc6 is 50, key is test_row_0/A:col10/1731569157296/Put/seqid=0 2024-11-14T07:25:57,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742168_1344 (size=13017) 2024-11-14T07:25:57,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742169_1345 (size=12301) 2024-11-14T07:25:57,843 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/8c1c24ecabd0476f969da9512a9aaf5f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/8c1c24ecabd0476f969da9512a9aaf5f 2024-11-14T07:25:57,848 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into 8c1c24ecabd0476f969da9512a9aaf5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:57,848 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:57,848 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569157764; duration=0sec 2024-11-14T07:25:57,848 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:57,848 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:57,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:57,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:57,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:57,850 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:57,850 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dc418dabbe2c4fa78f7486de10821c7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.6 K 2024-11-14T07:25:57,850 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc418dabbe2c4fa78f7486de10821c7c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569156274 2024-11-14T07:25:57,850 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41798215fb3b425da2bdf157fded1255, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731569156604 2024-11-14T07:25:57,851 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e962439f7f8481eae2ddb4aff937750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:57,868 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:57,868 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/4f1045bbd7d3484aa8ffdb6439a73d1f is 50, key is test_row_0/C:col10/1731569157287/Put/seqid=0 2024-11-14T07:25:57,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742170_1346 (size=13017) 2024-11-14T07:25:57,886 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/4f1045bbd7d3484aa8ffdb6439a73d1f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f1045bbd7d3484aa8ffdb6439a73d1f 2024-11-14T07:25:57,893 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into 4f1045bbd7d3484aa8ffdb6439a73d1f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:57,893 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:57,893 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569157764; duration=0sec 2024-11-14T07:25:57,893 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:57,893 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:25:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:57,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:25:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569217948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569217951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569217951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569217952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569217952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:57,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569218054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569218064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569218064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569218064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569218064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,221 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c0eb736613c342c88c06834901361919 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0eb736613c342c88c06834901361919 2024-11-14T07:25:58,227 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into c0eb736613c342c88c06834901361919(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:58,227 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:58,227 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569157764; duration=0sec 2024-11-14T07:25:58,227 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:58,227 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:58,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/22a7d2bbcd704f4b80c487f864c84cc6 2024-11-14T07:25:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c867b94b1f4342fa81b6511fe50c519b is 50, key is test_row_0/B:col10/1731569157296/Put/seqid=0 2024-11-14T07:25:58,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569218265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569218275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569218275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569218275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569218275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742171_1347 (size=12301) 2024-11-14T07:25:58,285 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c867b94b1f4342fa81b6511fe50c519b 2024-11-14T07:25:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ee406992eb9442a5bdeda7c844de08c9 is 50, key is test_row_0/C:col10/1731569157296/Put/seqid=0 2024-11-14T07:25:58,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742172_1348 (size=12301) 2024-11-14T07:25:58,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ee406992eb9442a5bdeda7c844de08c9 2024-11-14T07:25:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/22a7d2bbcd704f4b80c487f864c84cc6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6 2024-11-14T07:25:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:25:58,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c867b94b1f4342fa81b6511fe50c519b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b 2024-11-14T07:25:58,349 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:25:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ee406992eb9442a5bdeda7c844de08c9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9 2024-11-14T07:25:58,355 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T07:25:58,356 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f7fcaf3f9e5b875de457322a3a36b0e3 in 576ms, sequenceid=315, compaction requested=false 2024-11-14T07:25:58,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:58,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T07:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T07:25:58,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T07:25:58,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-11-14T07:25:58,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0450 sec 2024-11-14T07:25:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T07:25:58,455 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:25:58,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:25:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-14T07:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:25:58,457 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:25:58,457 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:25:58,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:25:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:25:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:58,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:25:58,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:25:58,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/38ed1aa666834736a099f4bbd5d11df6 is 50, key is test_row_0/A:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:58,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742173_1349 (size=12301) 2024-11-14T07:25:58,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569218589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569218592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569218592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569218593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569218595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569218704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569218704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569218705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569218705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569218705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:25:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569218915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569218915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569218915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569218915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569218916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:58,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:58,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/38ed1aa666834736a099f4bbd5d11df6 2024-11-14T07:25:59,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bd6067b9c5cc4b55a076463d8177a0ed is 50, key is test_row_0/B:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:59,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742174_1350 (size=12301) 2024-11-14T07:25:59,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bd6067b9c5cc4b55a076463d8177a0ed 2024-11-14T07:25:59,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:25:59,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/aed3379e150f422ba6ec3fdf12bfdb1b is 50, key is test_row_0/C:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742175_1351 (size=12301) 2024-11-14T07:25:59,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569219225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569219225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569219225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569219225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569219225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:25:59,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/aed3379e150f422ba6ec3fdf12bfdb1b 2024-11-14T07:25:59,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/38ed1aa666834736a099f4bbd5d11df6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6 2024-11-14T07:25:59,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T07:25:59,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/bd6067b9c5cc4b55a076463d8177a0ed as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed 2024-11-14T07:25:59,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T07:25:59,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/aed3379e150f422ba6ec3fdf12bfdb1b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b 2024-11-14T07:25:59,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T07:25:59,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for f7fcaf3f9e5b875de457322a3a36b0e3 in 943ms, sequenceid=341, compaction requested=true 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:59,521 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:59,521 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:25:59,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:59,521 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:59,521 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:25:59,522 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,522 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,522 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/8c1c24ecabd0476f969da9512a9aaf5f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.7 K 2024-11-14T07:25:59,522 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0eb736613c342c88c06834901361919, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.7 K 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1c24ecabd0476f969da9512a9aaf5f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c0eb736613c342c88c06834901361919, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22a7d2bbcd704f4b80c487f864c84cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731569157296 2024-11-14T07:25:59,522 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c867b94b1f4342fa81b6511fe50c519b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731569157296 2024-11-14T07:25:59,523 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ed1aa666834736a099f4bbd5d11df6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:25:59,523 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6067b9c5cc4b55a076463d8177a0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:25:59,528 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:59,529 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/cfe21dd9a8d94b5d927f5219d081f6f8 is 50, key is test_row_0/A:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:59,531 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:59,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-14T07:25:59,532 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/87bd66f09d0e4ca4834b958c2a07dc8d is 50, key is test_row_0/B:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,532 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:25:59,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742176_1352 (size=13119) 2024-11-14T07:25:59,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742177_1353 (size=13119) 2024-11-14T07:25:59,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/4d4cd3da2e9348f481aca739d07b00a7 is 50, key is test_row_0/A:col10/1731569158591/Put/seqid=0 2024-11-14T07:25:59,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742178_1354 (size=12301) 2024-11-14T07:25:59,554 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/4d4cd3da2e9348f481aca739d07b00a7 2024-11-14T07:25:59,554 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/87bd66f09d0e4ca4834b958c2a07dc8d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/87bd66f09d0e4ca4834b958c2a07dc8d 2024-11-14T07:25:59,559 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 87bd66f09d0e4ca4834b958c2a07dc8d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:59,559 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:59,559 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569159521; duration=0sec 2024-11-14T07:25:59,560 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:25:59,560 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:25:59,560 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:25:59,560 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:25:59,561 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:25:59,561 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:25:59,561 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f1045bbd7d3484aa8ffdb6439a73d1f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.7 K 2024-11-14T07:25:59,561 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1045bbd7d3484aa8ffdb6439a73d1f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731569156964 2024-11-14T07:25:59,561 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ee406992eb9442a5bdeda7c844de08c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731569157296 2024-11-14T07:25:59,561 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting aed3379e150f422ba6ec3fdf12bfdb1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:25:59,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/55f1744af9ba43799c727fa020b6d161 is 50, key is test_row_0/B:col10/1731569158591/Put/seqid=0 2024-11-14T07:25:59,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742179_1355 (size=12301) 2024-11-14T07:25:59,581 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:25:59,582 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/304b159ada244b65a329118040abbcec is 50, key is test_row_0/C:col10/1731569157950/Put/seqid=0 2024-11-14T07:25:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:25:59,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742180_1356 (size=13119) 2024-11-14T07:25:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:25:59,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:25:59,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569219750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569219753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569219753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569219754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569219754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569219865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569219865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569219865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569219867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:25:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569219867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:25:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:25:59,945 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/cfe21dd9a8d94b5d927f5219d081f6f8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cfe21dd9a8d94b5d927f5219d081f6f8 2024-11-14T07:25:59,950 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into cfe21dd9a8d94b5d927f5219d081f6f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:25:59,951 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:25:59,951 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569159521; duration=0sec 2024-11-14T07:25:59,951 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:25:59,951 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:25:59,978 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/55f1744af9ba43799c727fa020b6d161 2024-11-14T07:25:59,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/5cf1c54b43c14dc69b64a19e7bf258d9 is 50, key is test_row_0/C:col10/1731569158591/Put/seqid=0 2024-11-14T07:26:00,013 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/304b159ada244b65a329118040abbcec as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/304b159ada244b65a329118040abbcec 2024-11-14T07:26:00,027 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into 304b159ada244b65a329118040abbcec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:00,027 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:00,027 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569159521; duration=0sec 2024-11-14T07:26:00,027 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:00,027 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:26:00,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742181_1357 (size=12301) 2024-11-14T07:26:00,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/5cf1c54b43c14dc69b64a19e7bf258d9 2024-11-14T07:26:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/4d4cd3da2e9348f481aca739d07b00a7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7 2024-11-14T07:26:00,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7, entries=150, sequenceid=355, filesize=12.0 K 2024-11-14T07:26:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/55f1744af9ba43799c727fa020b6d161 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161 2024-11-14T07:26:00,051 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161, entries=150, sequenceid=355, filesize=12.0 K 2024-11-14T07:26:00,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/5cf1c54b43c14dc69b64a19e7bf258d9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9 2024-11-14T07:26:00,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9, entries=150, sequenceid=355, filesize=12.0 K 2024-11-14T07:26:00,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for f7fcaf3f9e5b875de457322a3a36b0e3 in 525ms, sequenceid=355, compaction requested=false 2024-11-14T07:26:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-14T07:26:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-14T07:26:00,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T07:26:00,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-14T07:26:00,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6070 sec 2024-11-14T07:26:00,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:00,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:00,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/96468eca486448aa889b3dd7eceea0d2 is 50, key is test_row_0/A:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742182_1358 (size=17181) 2024-11-14T07:26:00,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/96468eca486448aa889b3dd7eceea0d2 2024-11-14T07:26:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569220086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569220086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569220088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569220089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569220092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fe69882a84554133b8160e6d7f26c697 is 50, key is test_row_0/B:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742183_1359 (size=12301) 2024-11-14T07:26:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569220194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569220194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569220194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569220205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569220205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569220404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569220404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569220404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569220415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569220415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fe69882a84554133b8160e6d7f26c697 2024-11-14T07:26:00,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/0182172fb2ae46a18049d4c9b5fd333c is 50, key is test_row_0/C:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742184_1360 (size=12301) 2024-11-14T07:26:00,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/0182172fb2ae46a18049d4c9b5fd333c 2024-11-14T07:26:00,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/96468eca486448aa889b3dd7eceea0d2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2 2024-11-14T07:26:00,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2, entries=250, sequenceid=383, filesize=16.8 K 2024-11-14T07:26:00,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fe69882a84554133b8160e6d7f26c697 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697 2024-11-14T07:26:00,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T07:26:00,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/0182172fb2ae46a18049d4c9b5fd333c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c 2024-11-14T07:26:00,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c, entries=150, sequenceid=383, filesize=12.0 K 2024-11-14T07:26:00,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7fcaf3f9e5b875de457322a3a36b0e3 in 506ms, sequenceid=383, compaction requested=true 2024-11-14T07:26:00,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:00,583 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:00,584 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:00,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:00,585 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:00,585 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:26:00,585 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,585 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cfe21dd9a8d94b5d927f5219d081f6f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=41.6 K 2024-11-14T07:26:00,585 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:00,586 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:26:00,586 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,586 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/87bd66f09d0e4ca4834b958c2a07dc8d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.8 K 2024-11-14T07:26:00,586 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe21dd9a8d94b5d927f5219d081f6f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:26:00,586 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bd66f09d0e4ca4834b958c2a07dc8d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:26:00,586 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4cd3da2e9348f481aca739d07b00a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731569158588 2024-11-14T07:26:00,587 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f1744af9ba43799c727fa020b6d161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731569158588 2024-11-14T07:26:00,587 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96468eca486448aa889b3dd7eceea0d2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159746 2024-11-14T07:26:00,587 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fe69882a84554133b8160e6d7f26c697, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159753 2024-11-14T07:26:00,606 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T07:26:00,607 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:00,607 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/168f6423f41b48be80a7c7d85c44a916 is 50, key is test_row_0/A:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-14T07:26:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:00,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:00,613 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:00,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:00,618 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:00,618 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fcda4c35459444f599c18975ae6cc13a is 50, key is test_row_0/B:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742186_1362 (size=13221) 2024-11-14T07:26:00,654 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/fcda4c35459444f599c18975ae6cc13a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcda4c35459444f599c18975ae6cc13a 2024-11-14T07:26:00,660 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into fcda4c35459444f599c18975ae6cc13a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:00,660 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:00,660 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=13, startTime=1731569160584; duration=0sec 2024-11-14T07:26:00,661 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:00,661 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:26:00,661 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742185_1361 (size=13221) 2024-11-14T07:26:00,665 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:00,665 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:26:00,665 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,665 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/304b159ada244b65a329118040abbcec, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=36.8 K 2024-11-14T07:26:00,665 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 304b159ada244b65a329118040abbcec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731569157950 2024-11-14T07:26:00,666 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf1c54b43c14dc69b64a19e7bf258d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731569158588 2024-11-14T07:26:00,666 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0182172fb2ae46a18049d4c9b5fd333c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159753 2024-11-14T07:26:00,676 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:00,676 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b366f60e7ee14750867a8748832f5463 is 50, key is test_row_0/C:col10/1731569160075/Put/seqid=0 2024-11-14T07:26:00,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742187_1363 (size=13221) 2024-11-14T07:26:00,693 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/b366f60e7ee14750867a8748832f5463 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b366f60e7ee14750867a8748832f5463 2024-11-14T07:26:00,701 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into b366f60e7ee14750867a8748832f5463(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:00,701 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:00,701 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=13, startTime=1731569160584; duration=0sec 2024-11-14T07:26:00,701 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:00,702 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:26:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:00,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:26:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:00,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9a290acee17649a1bc0973895b8d32a1 is 50, key is test_row_0/A:col10/1731569160715/Put/seqid=0 2024-11-14T07:26:00,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742188_1364 (size=14741) 2024-11-14T07:26:00,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569220750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569220751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569220752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569220756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569220758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:00,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:00,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569220865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569220865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569220865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569220865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569220865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:00,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:00,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:00,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:00,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:01,073 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/168f6423f41b48be80a7c7d85c44a916 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/168f6423f41b48be80a7c7d85c44a916 2024-11-14T07:26:01,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569221074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569221074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569221075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569221075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569221075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,081 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into 168f6423f41b48be80a7c7d85c44a916(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:01,081 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:01,081 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=13, startTime=1731569160583; duration=0sec 2024-11-14T07:26:01,081 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:01,081 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:26:01,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9a290acee17649a1bc0973895b8d32a1 2024-11-14T07:26:01,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/2e280fa755c047168fbaa0435e604969 is 50, key is test_row_0/B:col10/1731569160715/Put/seqid=0 2024-11-14T07:26:01,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742189_1365 (size=12301) 2024-11-14T07:26:01,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:01,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569221384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569221384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569221384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569221385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569221385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:01,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/2e280fa755c047168fbaa0435e604969 2024-11-14T07:26:01,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/94fdd5f47b6f42d28206d53aef8ee669 is 50, key is test_row_0/C:col10/1731569160715/Put/seqid=0 2024-11-14T07:26:01,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742190_1366 (size=12301) 2024-11-14T07:26:01,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/94fdd5f47b6f42d28206d53aef8ee669 2024-11-14T07:26:01,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/9a290acee17649a1bc0973895b8d32a1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1 2024-11-14T07:26:01,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1, entries=200, sequenceid=397, filesize=14.4 K 2024-11-14T07:26:01,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/2e280fa755c047168fbaa0435e604969 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969 2024-11-14T07:26:01,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969, entries=150, sequenceid=397, filesize=12.0 K 2024-11-14T07:26:01,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/94fdd5f47b6f42d28206d53aef8ee669 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669 2024-11-14T07:26:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669, entries=150, sequenceid=397, filesize=12.0 K 2024-11-14T07:26:01,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7fcaf3f9e5b875de457322a3a36b0e3 in 930ms, sequenceid=397, compaction requested=false 2024-11-14T07:26:01,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:01,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-14T07:26:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:01,689 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:26:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/cef0723e97094103bd76212ecea4c530 is 50, key is test_row_0/A:col10/1731569160749/Put/seqid=0 2024-11-14T07:26:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742191_1367 (size=12301) 2024-11-14T07:26:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:01,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:01,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569221903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569221904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569221907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569221909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569221909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569222014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569222014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569222024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569222024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/cef0723e97094103bd76212ecea4c530 2024-11-14T07:26:02,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/17130fffa9a94553bff549235d41d13f is 50, key is test_row_0/B:col10/1731569160749/Put/seqid=0 2024-11-14T07:26:02,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742192_1368 (size=12301) 2024-11-14T07:26:02,144 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/17130fffa9a94553bff549235d41d13f 2024-11-14T07:26:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ecf2817910d842118c4212be551a1109 is 50, key is test_row_0/C:col10/1731569160749/Put/seqid=0 2024-11-14T07:26:02,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742193_1369 (size=12301) 2024-11-14T07:26:02,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569222224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569222224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569222235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569222235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569222534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569222534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569222544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569222544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:02,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ecf2817910d842118c4212be551a1109 2024-11-14T07:26:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/cef0723e97094103bd76212ecea4c530 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530 2024-11-14T07:26:02,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T07:26:02,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/17130fffa9a94553bff549235d41d13f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f 2024-11-14T07:26:02,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T07:26:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/ecf2817910d842118c4212be551a1109 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109 2024-11-14T07:26:02,585 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T07:26:02,586 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f7fcaf3f9e5b875de457322a3a36b0e3 in 898ms, sequenceid=422, compaction requested=true 2024-11-14T07:26:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:02,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-14T07:26:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-14T07:26:02,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T07:26:02,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9760 sec 2024-11-14T07:26:02,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.9820 sec 2024-11-14T07:26:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-14T07:26:02,754 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:02,755 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-14T07:26:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T07:26:02,756 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:02,757 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:02,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T07:26:02,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-14T07:26:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:02,911 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:26:02,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/af2f50f370614e568a6179d960edb78a is 50, key is test_row_0/A:col10/1731569161907/Put/seqid=0 2024-11-14T07:26:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:02,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:02,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742194_1370 (size=12301) 2024-11-14T07:26:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569223022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569223044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569223044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569223055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569223055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T07:26:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569223135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/af2f50f370614e568a6179d960edb78a 2024-11-14T07:26:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/912d723d7a634cf3b377807a0ee429db is 50, key is test_row_0/B:col10/1731569161907/Put/seqid=0 2024-11-14T07:26:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569223344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742195_1371 (size=12301) 2024-11-14T07:26:03,362 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/912d723d7a634cf3b377807a0ee429db 2024-11-14T07:26:03,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9d515f664713449992dcc7af8833ca47 is 50, key is test_row_0/C:col10/1731569161907/Put/seqid=0 2024-11-14T07:26:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T07:26:03,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742196_1372 (size=12301) 2024-11-14T07:26:03,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569223655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:03,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9d515f664713449992dcc7af8833ca47 2024-11-14T07:26:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/af2f50f370614e568a6179d960edb78a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a 2024-11-14T07:26:03,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T07:26:03,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/912d723d7a634cf3b377807a0ee429db as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db 2024-11-14T07:26:03,802 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T07:26:03,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/9d515f664713449992dcc7af8833ca47 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47 2024-11-14T07:26:03,807 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47, entries=150, sequenceid=435, filesize=12.0 K 2024-11-14T07:26:03,808 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7fcaf3f9e5b875de457322a3a36b0e3 in 897ms, sequenceid=435, compaction requested=true 2024-11-14T07:26:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-14T07:26:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-14T07:26:03,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T07:26:03,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-11-14T07:26:03,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0580 sec 2024-11-14T07:26:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T07:26:03,894 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:03,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-14T07:26:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:26:03,896 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:03,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:03,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:26:04,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-14T07:26:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:04,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:26:04,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/18f41da7eea44a4296e44b4d9dc1e142 is 50, key is test_row_0/A:col10/1731569162991/Put/seqid=0 2024-11-14T07:26:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742197_1373 (size=12301) 2024-11-14T07:26:04,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/18f41da7eea44a4296e44b4d9dc1e142 2024-11-14T07:26:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c0e0c194d44f4429aab06b547a088705 is 50, key is test_row_0/B:col10/1731569162991/Put/seqid=0 2024-11-14T07:26:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742198_1374 (size=12301) 2024-11-14T07:26:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569224074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569224074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569224076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,079 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in f7fcaf3f9e5b875de457322a3a36b0e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569224079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569224164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569224184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569224184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569224185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:26:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569224394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569224394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569224395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c0e0c194d44f4429aab06b547a088705 2024-11-14T07:26:04,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f473f59b847e4fd18fe8268d5003a3c5 is 50, key is test_row_0/C:col10/1731569162991/Put/seqid=0 2024-11-14T07:26:04,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742199_1375 (size=12301) 2024-11-14T07:26:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:26:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:04,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f473f59b847e4fd18fe8268d5003a3c5 2024-11-14T07:26:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/18f41da7eea44a4296e44b4d9dc1e142 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142 2024-11-14T07:26:04,888 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T07:26:04,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/c0e0c194d44f4429aab06b547a088705 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705 2024-11-14T07:26:04,893 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T07:26:04,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/f473f59b847e4fd18fe8268d5003a3c5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5 2024-11-14T07:26:04,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T07:26:04,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7fcaf3f9e5b875de457322a3a36b0e3 in 849ms, sequenceid=458, compaction requested=true 2024-11-14T07:26:04,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:04,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:04,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-14T07:26:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-14T07:26:04,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T07:26:04,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-14T07:26:04,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.0060 sec 2024-11-14T07:26:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-14T07:26:05,035 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:05,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-14T07:26:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:05,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:05,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:05,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:05,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-14T07:26:05,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:05,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:26:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:05,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. as already flushing 2024-11-14T07:26:05,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0bc5484bb643442fb21871f5bcd9bf3a is 50, key is test_row_0/A:col10/1731569165187/Put/seqid=0 2024-11-14T07:26:05,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742200_1376 (size=12301) 2024-11-14T07:26:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569225262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569225290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569225290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569225290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569225394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569225404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569225404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569225404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,558 INFO [Thread-1291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1291. 2024-11-14T07:26:05,558 DEBUG [Thread-1291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:05,558 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,558 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:05,558 INFO [Thread-1293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1293. 2024-11-14T07:26:05,558 DEBUG [Thread-1293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:05,558 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,558 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:05,560 INFO [Thread-1285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1285. 2024-11-14T07:26:05,560 DEBUG [Thread-1285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:05,560 DEBUG [Thread-1285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,560 DEBUG [Thread-1285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:05,562 INFO [Thread-1289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1289. 2024-11-14T07:26:05,562 DEBUG [Thread-1289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:05,562 DEBUG [Thread-1289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,562 DEBUG [Thread-1289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:05,563 INFO [Thread-1287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1287. 2024-11-14T07:26:05,563 DEBUG [Thread-1287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:05,563 DEBUG [Thread-1287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,563 DEBUG [Thread-1287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:05,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:05,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569225604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569225615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569225615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569225615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,647 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0bc5484bb643442fb21871f5bcd9bf3a 2024-11-14T07:26:05,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1731b5f8a43e4da490bab4f14ae052a9 is 50, key is test_row_0/B:col10/1731569165187/Put/seqid=0 2024-11-14T07:26:05,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742201_1377 (size=12301) 2024-11-14T07:26:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:05,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569225914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569225924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569225925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569225925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,066 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1731b5f8a43e4da490bab4f14ae052a9 2024-11-14T07:26:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/2cf97465e09d4ebba4e82994c3d2f9f1 is 50, key is test_row_0/C:col10/1731569165187/Put/seqid=0 2024-11-14T07:26:06,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742202_1378 (size=12301) 2024-11-14T07:26:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1731569226105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:06,105 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in f7fcaf3f9e5b875de457322a3a36b0e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58026 deadline: 1731569226425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1731569226435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:06,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1731569226435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:06,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57992 deadline: 1731569226435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:06,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/2cf97465e09d4ebba4e82994c3d2f9f1 2024-11-14T07:26:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/0bc5484bb643442fb21871f5bcd9bf3a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a 2024-11-14T07:26:06,499 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T07:26:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/1731b5f8a43e4da490bab4f14ae052a9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9 2024-11-14T07:26:06,506 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T07:26:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/2cf97465e09d4ebba4e82994c3d2f9f1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1 2024-11-14T07:26:06,513 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1, entries=150, sequenceid=472, filesize=12.0 K 2024-11-14T07:26:06,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1325ms, sequenceid=472, compaction requested=true 2024-11-14T07:26:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:06,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:06,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-14T07:26:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-14T07:26:06,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T07:26:06,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-14T07:26:06,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.4840 sec 2024-11-14T07:26:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T07:26:07,186 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:07,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:07,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:07,452 INFO [Thread-1280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1280. 2024-11-14T07:26:07,452 INFO [Thread-1274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1274. 2024-11-14T07:26:07,452 INFO [Thread-1282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1282. 2024-11-14T07:26:07,452 DEBUG [Thread-1280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:07,452 DEBUG [Thread-1274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:07,452 DEBUG [Thread-1282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:07,452 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,452 DEBUG [Thread-1274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,452 DEBUG [Thread-1282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,453 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,453 DEBUG [Thread-1274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,453 DEBUG [Thread-1282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:07,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:07,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:07,456 INFO [Thread-1276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1276. 2024-11-14T07:26:07,456 DEBUG [Thread-1276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:07,456 DEBUG [Thread-1276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,456 DEBUG [Thread-1276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:07,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:07,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/ae30cfc0bce74618bb83692b4712f405 is 50, key is test_row_0/A:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:07,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742203_1379 (size=12301) 2024-11-14T07:26:07,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/ae30cfc0bce74618bb83692b4712f405 2024-11-14T07:26:07,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/b97bd4a7067b4bb199cba0ad090effcb is 50, key is test_row_0/B:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:07,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742204_1380 (size=12301) 2024-11-14T07:26:08,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/b97bd4a7067b4bb199cba0ad090effcb 2024-11-14T07:26:08,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/47b2fc2c4bdd419d85ff37199661f6d2 is 50, key is test_row_0/C:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:08,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742205_1381 (size=12301) 2024-11-14T07:26:08,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/47b2fc2c4bdd419d85ff37199661f6d2 2024-11-14T07:26:08,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/ae30cfc0bce74618bb83692b4712f405 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405 2024-11-14T07:26:08,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405, entries=150, sequenceid=497, filesize=12.0 K 2024-11-14T07:26:08,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/b97bd4a7067b4bb199cba0ad090effcb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb 2024-11-14T07:26:08,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb, entries=150, sequenceid=497, filesize=12.0 K 2024-11-14T07:26:08,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/47b2fc2c4bdd419d85ff37199661f6d2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2 2024-11-14T07:26:08,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2, entries=150, sequenceid=497, filesize=12.0 K 2024-11-14T07:26:08,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1270ms, sequenceid=497, compaction requested=true 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:08,722 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:08,722 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7fcaf3f9e5b875de457322a3a36b0e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:08,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:08,724 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 87027 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T07:26:08,724 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 89467 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T07:26:08,724 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/B is initiating minor compaction (all files) 2024-11-14T07:26:08,724 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/A is initiating minor compaction (all files) 2024-11-14T07:26:08,724 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/B in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:08,724 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/A in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:08,725 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcda4c35459444f599c18975ae6cc13a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=85.0 K 2024-11-14T07:26:08,725 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/168f6423f41b48be80a7c7d85c44a916, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=87.4 K 2024-11-14T07:26:08,725 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168f6423f41b48be80a7c7d85c44a916, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159753 2024-11-14T07:26:08,725 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fcda4c35459444f599c18975ae6cc13a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159753 2024-11-14T07:26:08,725 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a290acee17649a1bc0973895b8d32a1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731569160078 2024-11-14T07:26:08,725 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e280fa755c047168fbaa0435e604969, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731569160078 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 17130fffa9a94553bff549235d41d13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731569160749 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef0723e97094103bd76212ecea4c530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731569160749 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 912d723d7a634cf3b377807a0ee429db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731569161902 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2f50f370614e568a6179d960edb78a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731569161902 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e0c194d44f4429aab06b547a088705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731569162991 2024-11-14T07:26:08,726 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f41da7eea44a4296e44b4d9dc1e142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731569162991 2024-11-14T07:26:08,727 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1731b5f8a43e4da490bab4f14ae052a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731569164072 2024-11-14T07:26:08,727 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc5484bb643442fb21871f5bcd9bf3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731569164072 2024-11-14T07:26:08,727 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b97bd4a7067b4bb199cba0ad090effcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731569165262 2024-11-14T07:26:08,727 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae30cfc0bce74618bb83692b4712f405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731569165262 2024-11-14T07:26:08,738 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#A#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:08,738 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:08,739 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/74717f8dc4e84fd5b15a8def57d04f9b is 50, key is test_row_0/B:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:08,739 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/d713565dfd564a49bf6b5f838eda9c04 is 50, key is test_row_0/A:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742207_1383 (size=13459) 2024-11-14T07:26:08,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742206_1382 (size=13459) 2024-11-14T07:26:09,147 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/d713565dfd564a49bf6b5f838eda9c04 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/d713565dfd564a49bf6b5f838eda9c04 2024-11-14T07:26:09,152 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/74717f8dc4e84fd5b15a8def57d04f9b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/74717f8dc4e84fd5b15a8def57d04f9b 2024-11-14T07:26:09,154 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/A of f7fcaf3f9e5b875de457322a3a36b0e3 into d713565dfd564a49bf6b5f838eda9c04(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:09,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:09,154 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/A, priority=9, startTime=1731569168722; duration=0sec 2024-11-14T07:26:09,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:09,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:A 2024-11-14T07:26:09,154 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 87027 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): f7fcaf3f9e5b875de457322a3a36b0e3/C is initiating minor compaction (all files) 2024-11-14T07:26:09,156 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7fcaf3f9e5b875de457322a3a36b0e3/C in TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:09,156 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b366f60e7ee14750867a8748832f5463, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp, totalSize=85.0 K 2024-11-14T07:26:09,156 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/B of f7fcaf3f9e5b875de457322a3a36b0e3 into 74717f8dc4e84fd5b15a8def57d04f9b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:09,156 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/B, priority=9, startTime=1731569168722; duration=0sec 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:B 2024-11-14T07:26:09,156 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b366f60e7ee14750867a8748832f5463, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731569159753 2024-11-14T07:26:09,157 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94fdd5f47b6f42d28206d53aef8ee669, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731569160078 2024-11-14T07:26:09,157 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf2817910d842118c4212be551a1109, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731569160749 2024-11-14T07:26:09,157 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d515f664713449992dcc7af8833ca47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731569161902 2024-11-14T07:26:09,157 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f473f59b847e4fd18fe8268d5003a3c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731569162991 2024-11-14T07:26:09,158 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf97465e09d4ebba4e82994c3d2f9f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731569164072 2024-11-14T07:26:09,158 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b2fc2c4bdd419d85ff37199661f6d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731569165262 2024-11-14T07:26:09,170 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7fcaf3f9e5b875de457322a3a36b0e3#C#compaction#334 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:09,170 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/494600e2fe4b4060a3e25d07782e8ad2 is 50, key is test_row_0/C:col10/1731569167447/Put/seqid=0 2024-11-14T07:26:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742208_1384 (size=13459) 2024-11-14T07:26:09,578 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/494600e2fe4b4060a3e25d07782e8ad2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/494600e2fe4b4060a3e25d07782e8ad2 2024-11-14T07:26:09,582 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in f7fcaf3f9e5b875de457322a3a36b0e3/C of f7fcaf3f9e5b875de457322a3a36b0e3 into 494600e2fe4b4060a3e25d07782e8ad2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:09,582 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7fcaf3f9e5b875de457322a3a36b0e3: 2024-11-14T07:26:09,582 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., storeName=f7fcaf3f9e5b875de457322a3a36b0e3/C, priority=9, startTime=1731569168722; duration=0sec 2024-11-14T07:26:09,582 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:09,582 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7fcaf3f9e5b875de457322a3a36b0e3:C 2024-11-14T07:26:10,145 INFO [Thread-1278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1278. 2024-11-14T07:26:10,146 DEBUG [Thread-1278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:10,146 DEBUG [Thread-1278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:10,146 DEBUG [Thread-1278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:10,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3957 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3948 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3947 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3952 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3923 2024-11-14T07:26:10,146 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:26:10,146 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:26:10,146 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:26:10,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:10,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:10,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:10,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:26:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:10,161 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569170161"}]},"ts":"1731569170161"} 2024-11-14T07:26:10,163 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:26:10,163 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:26:10,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:26:10,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, UNASSIGN}] 2024-11-14T07:26:10,166 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, UNASSIGN 2024-11-14T07:26:10,167 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f7fcaf3f9e5b875de457322a3a36b0e3, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:10,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, UNASSIGN because future has completed 2024-11-14T07:26:10,168 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:26:10,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:10,321 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:10,321 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:26:10,321 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing f7fcaf3f9e5b875de457322a3a36b0e3, disabling compactions & flushes 2024-11-14T07:26:10,321 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:10,321 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:10,321 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. after waiting 0 ms 2024-11-14T07:26:10,321 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:10,321 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing f7fcaf3f9e5b875de457322a3a36b0e3 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-14T07:26:10,325 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c6ed0f0f512649e5aeb8a2f24743f991 is 50, key is test_row_0/A:col10/1731569170144/Put/seqid=0 2024-11-14T07:26:10,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742209_1385 (size=9857) 2024-11-14T07:26:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:10,729 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c6ed0f0f512649e5aeb8a2f24743f991 2024-11-14T07:26:10,736 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/d6bee4c3fe914fd697a57441d109451b is 50, key is test_row_0/B:col10/1731569170144/Put/seqid=0 2024-11-14T07:26:10,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742210_1386 (size=9857) 2024-11-14T07:26:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:11,142 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/d6bee4c3fe914fd697a57441d109451b 2024-11-14T07:26:11,147 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/c0b66e611f0d42e69b502f02e5466905 is 50, key is test_row_0/C:col10/1731569170144/Put/seqid=0 2024-11-14T07:26:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742211_1387 (size=9857) 2024-11-14T07:26:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:11,551 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/c0b66e611f0d42e69b502f02e5466905 2024-11-14T07:26:11,557 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/A/c6ed0f0f512649e5aeb8a2f24743f991 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c6ed0f0f512649e5aeb8a2f24743f991 2024-11-14T07:26:11,561 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c6ed0f0f512649e5aeb8a2f24743f991, entries=100, sequenceid=505, filesize=9.6 K 2024-11-14T07:26:11,562 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/B/d6bee4c3fe914fd697a57441d109451b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/d6bee4c3fe914fd697a57441d109451b 2024-11-14T07:26:11,566 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/d6bee4c3fe914fd697a57441d109451b, entries=100, sequenceid=505, filesize=9.6 K 2024-11-14T07:26:11,567 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/.tmp/C/c0b66e611f0d42e69b502f02e5466905 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/c0b66e611f0d42e69b502f02e5466905 2024-11-14T07:26:11,571 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/c0b66e611f0d42e69b502f02e5466905, entries=100, sequenceid=505, filesize=9.6 K 2024-11-14T07:26:11,572 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1251ms, sequenceid=505, compaction requested=false 2024-11-14T07:26:11,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/bd4a0179cf6b4011b218c46fb69eae55, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/dad5b966f0fd44ad9d7a01e7276fad47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/872f3bf4d9604620ad06534b1c3159cd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/eeba4529b2b64160a8accddf1106fd81, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e6740e73acb5426591102ca0fbc7d86c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2d0a19e3bb244c279fe481924384d87c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/8c1c24ecabd0476f969da9512a9aaf5f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cfe21dd9a8d94b5d927f5219d081f6f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/168f6423f41b48be80a7c7d85c44a916, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405] to archive 2024-11-14T07:26:11,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:11,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/da84ddac163a4d0eb441817dc5c03f7a 2024-11-14T07:26:11,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/a1cd21b30d6e477e9fdafe4eaeb5232d 2024-11-14T07:26:11,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/7226e3455d1341a98b1bf15f593f93ed 2024-11-14T07:26:11,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/bd4a0179cf6b4011b218c46fb69eae55 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/bd4a0179cf6b4011b218c46fb69eae55 2024-11-14T07:26:11,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0144ef0dcd904f24a99e27a95e973a59 2024-11-14T07:26:11,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/dad5b966f0fd44ad9d7a01e7276fad47 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/dad5b966f0fd44ad9d7a01e7276fad47 2024-11-14T07:26:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/3bbb7b1db7ab4ebb9c6b915701163fa7 2024-11-14T07:26:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e2fc7841cb844ba599dae28cc2756576 2024-11-14T07:26:11,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9fb8294804494cffbde5a333341c3ad5 2024-11-14T07:26:11,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/872f3bf4d9604620ad06534b1c3159cd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/872f3bf4d9604620ad06534b1c3159cd 2024-11-14T07:26:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2cf77571afa041e7b7c71cc23c126b86 2024-11-14T07:26:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/eeba4529b2b64160a8accddf1106fd81 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/eeba4529b2b64160a8accddf1106fd81 2024-11-14T07:26:11,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/14a4746aa7fa4bd8952d2e1df2e459f6 2024-11-14T07:26:11,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c09a5e6aa59e49e0a32d234088d88f65 2024-11-14T07:26:11,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e6740e73acb5426591102ca0fbc7d86c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e6740e73acb5426591102ca0fbc7d86c 2024-11-14T07:26:11,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/90f3dc359fb3476ea9f1b553d11f77b6 2024-11-14T07:26:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/e33c53e93ae942f58ea91ead9b8538e8 2024-11-14T07:26:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2d0a19e3bb244c279fe481924384d87c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/2d0a19e3bb244c279fe481924384d87c 2024-11-14T07:26:11,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/b1eb1f5e5e89427dad3dfc8e256d070f 2024-11-14T07:26:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c9780be57f4142dc88f25fab3ff0cd1b 2024-11-14T07:26:11,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/64c121b20ecf4e678f176f56c22fcb9a 2024-11-14T07:26:11,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/8c1c24ecabd0476f969da9512a9aaf5f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/8c1c24ecabd0476f969da9512a9aaf5f 2024-11-14T07:26:11,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/22a7d2bbcd704f4b80c487f864c84cc6 2024-11-14T07:26:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cfe21dd9a8d94b5d927f5219d081f6f8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cfe21dd9a8d94b5d927f5219d081f6f8 2024-11-14T07:26:11,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/38ed1aa666834736a099f4bbd5d11df6 2024-11-14T07:26:11,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/4d4cd3da2e9348f481aca739d07b00a7 2024-11-14T07:26:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/96468eca486448aa889b3dd7eceea0d2 2024-11-14T07:26:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/168f6423f41b48be80a7c7d85c44a916 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/168f6423f41b48be80a7c7d85c44a916 2024-11-14T07:26:11,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/9a290acee17649a1bc0973895b8d32a1 2024-11-14T07:26:11,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/cef0723e97094103bd76212ecea4c530 2024-11-14T07:26:11,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/af2f50f370614e568a6179d960edb78a 2024-11-14T07:26:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/18f41da7eea44a4296e44b4d9dc1e142 2024-11-14T07:26:11,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/0bc5484bb643442fb21871f5bcd9bf3a 2024-11-14T07:26:11,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/ae30cfc0bce74618bb83692b4712f405 2024-11-14T07:26:11,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1e2d9b7980e44fe1a470aba66dca0f6b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/3280d5688ccc4c608bc845cd6fa0ec7e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5968db5713554267ab7f4c9b4c6041c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/6d1463b2a9aa407289eb2480a7fc8dde, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcc48e340e3448c0a9817ef79787526a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7083b770e9b849278a5ba2de1a0bf22d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0eb736613c342c88c06834901361919, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/87bd66f09d0e4ca4834b958c2a07dc8d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcda4c35459444f599c18975ae6cc13a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb] to archive 2024-11-14T07:26:11,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7c106c6a4ead4c70a3ef45c9c4d97ace 2024-11-14T07:26:11,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/be2f0b4f350d41c5a5eb6659425d1731 2024-11-14T07:26:11,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1e2d9b7980e44fe1a470aba66dca0f6b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1e2d9b7980e44fe1a470aba66dca0f6b 2024-11-14T07:26:11,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/0481d4675fd547f28a9139059c53abc6 2024-11-14T07:26:11,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bead5cb30a154359916fb16c110cd8c0 2024-11-14T07:26:11,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/3280d5688ccc4c608bc845cd6fa0ec7e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/3280d5688ccc4c608bc845cd6fa0ec7e 2024-11-14T07:26:11,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/708afb7d40a04df99ec44776cd84491e 2024-11-14T07:26:11,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/41657b8c88d04c06b793f6e45b9153b1 2024-11-14T07:26:11,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5968db5713554267ab7f4c9b4c6041c5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5968db5713554267ab7f4c9b4c6041c5 2024-11-14T07:26:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/8cbe5612572348d4ad44a795d3ce3e2c 2024-11-14T07:26:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1913b2a8c46143f6b200d8ff32f0eb29 2024-11-14T07:26:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/6d1463b2a9aa407289eb2480a7fc8dde to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/6d1463b2a9aa407289eb2480a7fc8dde 2024-11-14T07:26:11,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/65e24a43edcd46aeb07f8e9b9d0fc031 2024-11-14T07:26:11,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/ca69d34e3fb14338adcfe1b87c4f52b0 2024-11-14T07:26:11,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcc48e340e3448c0a9817ef79787526a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcc48e340e3448c0a9817ef79787526a 2024-11-14T07:26:11,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/e12587f108d14594bed5e3d201215e70 2024-11-14T07:26:11,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/a21ffc75cf3c4b56a86c64ca43a77748 2024-11-14T07:26:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7083b770e9b849278a5ba2de1a0bf22d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/7083b770e9b849278a5ba2de1a0bf22d 2024-11-14T07:26:11,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/5fb11acb9392422d98602cf8597dbdf1 2024-11-14T07:26:11,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/caed516d3c8c4ec6b3d8b13848afb48a 2024-11-14T07:26:11,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0eb736613c342c88c06834901361919 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0eb736613c342c88c06834901361919 2024-11-14T07:26:11,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/34179b4cd8904ed9b34007bd8438b74c 2024-11-14T07:26:11,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c867b94b1f4342fa81b6511fe50c519b 2024-11-14T07:26:11,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/87bd66f09d0e4ca4834b958c2a07dc8d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/87bd66f09d0e4ca4834b958c2a07dc8d 2024-11-14T07:26:11,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/bd6067b9c5cc4b55a076463d8177a0ed 2024-11-14T07:26:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/55f1744af9ba43799c727fa020b6d161 2024-11-14T07:26:11,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcda4c35459444f599c18975ae6cc13a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fcda4c35459444f599c18975ae6cc13a 2024-11-14T07:26:11,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/fe69882a84554133b8160e6d7f26c697 2024-11-14T07:26:11,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/2e280fa755c047168fbaa0435e604969 2024-11-14T07:26:11,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/17130fffa9a94553bff549235d41d13f 2024-11-14T07:26:11,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/912d723d7a634cf3b377807a0ee429db 2024-11-14T07:26:11,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/c0e0c194d44f4429aab06b547a088705 2024-11-14T07:26:11,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/1731b5f8a43e4da490bab4f14ae052a9 2024-11-14T07:26:11,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/b97bd4a7067b4bb199cba0ad090effcb 2024-11-14T07:26:11,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d84a36bb20884e6abcfe85f5447f4420, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/52e78fed6c674e0aaffdda2a47e22539, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/da016d29298248f69de79500739c2795, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad5ec1f54aea45bdb6e346d5ac53bc64, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b106ddf5c98c40ffb6e60a356b70bbd3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dc418dabbe2c4fa78f7486de10821c7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f1045bbd7d3484aa8ffdb6439a73d1f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/304b159ada244b65a329118040abbcec, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b366f60e7ee14750867a8748832f5463, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2] to archive 2024-11-14T07:26:11,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:11,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f4536f8d11247f482faa22586141ddb 2024-11-14T07:26:11,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/424e7ed8341243a1bf2d152f9102b905 2024-11-14T07:26:11,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d84a36bb20884e6abcfe85f5447f4420 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d84a36bb20884e6abcfe85f5447f4420 2024-11-14T07:26:11,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad437fa8a7fe43979287eae29a2929bc 2024-11-14T07:26:11,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f1b4af233c4c4fd5837e504ee4261a5f 2024-11-14T07:26:11,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/52e78fed6c674e0aaffdda2a47e22539 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/52e78fed6c674e0aaffdda2a47e22539 2024-11-14T07:26:11,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b5fc626188c74f4db0825a83b22b62fe 2024-11-14T07:26:11,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/caa3720547a1402f8cd098837343f3f0 2024-11-14T07:26:11,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/da016d29298248f69de79500739c2795 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/da016d29298248f69de79500739c2795 2024-11-14T07:26:11,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/6ceab01002e741f8b3ce8b06e9d6da71 2024-11-14T07:26:11,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/15c222d59b904568981be6c087ab55ba 2024-11-14T07:26:11,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad5ec1f54aea45bdb6e346d5ac53bc64 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ad5ec1f54aea45bdb6e346d5ac53bc64 2024-11-14T07:26:11,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/530bdeffb6f643babe1658f5c9cc9747 2024-11-14T07:26:11,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/d342a65d498a460f9903559e41109dd3 2024-11-14T07:26:11,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b106ddf5c98c40ffb6e60a356b70bbd3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b106ddf5c98c40ffb6e60a356b70bbd3 2024-11-14T07:26:11,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/a9a1666ec39144118a462c322855bc0a 2024-11-14T07:26:11,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/e417ebbe351b475e9403f414c3783106 2024-11-14T07:26:11,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dc418dabbe2c4fa78f7486de10821c7c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dc418dabbe2c4fa78f7486de10821c7c 2024-11-14T07:26:11,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/dcb6a83848a54ae7b63e214520c2f6ae 2024-11-14T07:26:11,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/41798215fb3b425da2bdf157fded1255 2024-11-14T07:26:11,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f1045bbd7d3484aa8ffdb6439a73d1f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/4f1045bbd7d3484aa8ffdb6439a73d1f 2024-11-14T07:26:11,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9e962439f7f8481eae2ddb4aff937750 2024-11-14T07:26:11,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ee406992eb9442a5bdeda7c844de08c9 2024-11-14T07:26:11,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/304b159ada244b65a329118040abbcec to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/304b159ada244b65a329118040abbcec 2024-11-14T07:26:11,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/aed3379e150f422ba6ec3fdf12bfdb1b 2024-11-14T07:26:11,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/5cf1c54b43c14dc69b64a19e7bf258d9 2024-11-14T07:26:11,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b366f60e7ee14750867a8748832f5463 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/b366f60e7ee14750867a8748832f5463 2024-11-14T07:26:11,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/0182172fb2ae46a18049d4c9b5fd333c 2024-11-14T07:26:11,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/94fdd5f47b6f42d28206d53aef8ee669 2024-11-14T07:26:11,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/ecf2817910d842118c4212be551a1109 2024-11-14T07:26:11,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/9d515f664713449992dcc7af8833ca47 2024-11-14T07:26:11,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/f473f59b847e4fd18fe8268d5003a3c5 2024-11-14T07:26:11,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/2cf97465e09d4ebba4e82994c3d2f9f1 2024-11-14T07:26:11,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/47b2fc2c4bdd419d85ff37199661f6d2 2024-11-14T07:26:11,670 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/recovered.edits/508.seqid, newMaxSeqId=508, maxSeqId=1 2024-11-14T07:26:11,670 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. 2024-11-14T07:26:11,670 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for f7fcaf3f9e5b875de457322a3a36b0e3: Waiting for close lock at 1731569170321Running coprocessor pre-close hooks at 1731569170321Disabling compacts and flushes for region at 1731569170321Disabling writes for close at 1731569170321Obtaining lock to block concurrent updates at 1731569170321Preparing flush snapshotting stores in f7fcaf3f9e5b875de457322a3a36b0e3 at 1731569170321Finished memstore snapshotting TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731569170322 (+1 ms)Flushing stores of TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3. at 1731569170322Flushing f7fcaf3f9e5b875de457322a3a36b0e3/A: creating writer at 1731569170322Flushing f7fcaf3f9e5b875de457322a3a36b0e3/A: appending metadata at 1731569170325 (+3 ms)Flushing f7fcaf3f9e5b875de457322a3a36b0e3/A: closing flushed file at 1731569170325Flushing f7fcaf3f9e5b875de457322a3a36b0e3/B: creating writer at 1731569170733 (+408 ms)Flushing f7fcaf3f9e5b875de457322a3a36b0e3/B: appending metadata at 1731569170735 (+2 ms)Flushing f7fcaf3f9e5b875de457322a3a36b0e3/B: closing flushed file at 1731569170735Flushing f7fcaf3f9e5b875de457322a3a36b0e3/C: creating writer at 1731569171145 (+410 ms)Flushing f7fcaf3f9e5b875de457322a3a36b0e3/C: appending metadata at 1731569171147 (+2 ms)Flushing f7fcaf3f9e5b875de457322a3a36b0e3/C: closing flushed file at 1731569171147Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b30580: reopening flushed file at 1731569171556 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f8392e: reopening flushed file at 1731569171561 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dfdecb1: reopening flushed file at 1731569171566 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for f7fcaf3f9e5b875de457322a3a36b0e3 in 1251ms, sequenceid=505, compaction requested=false at 1731569171572 (+6 ms)Writing region close event to WAL at 1731569171667 (+95 ms)Running coprocessor post-close hooks at 1731569171670 (+3 ms)Closed at 1731569171670 2024-11-14T07:26:11,672 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:11,673 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=f7fcaf3f9e5b875de457322a3a36b0e3, regionState=CLOSED 2024-11-14T07:26:11,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:11,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-14T07:26:11,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure f7fcaf3f9e5b875de457322a3a36b0e3, server=c7aa6e2790b3,46235,1731569085523 in 1.5070 sec 2024-11-14T07:26:11,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T07:26:11,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7fcaf3f9e5b875de457322a3a36b0e3, UNASSIGN in 1.5120 sec 2024-11-14T07:26:11,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T07:26:11,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-14T07:26:11,684 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569171683"}]},"ts":"1731569171683"} 2024-11-14T07:26:11,685 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:26:11,685 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:26:11,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-14T07:26:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-14T07:26:12,306 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:12,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:26:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,310 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-14T07:26:12,312 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:12,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/recovered.edits] 2024-11-14T07:26:12,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c6ed0f0f512649e5aeb8a2f24743f991 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/c6ed0f0f512649e5aeb8a2f24743f991 2024-11-14T07:26:12,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/d713565dfd564a49bf6b5f838eda9c04 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/A/d713565dfd564a49bf6b5f838eda9c04 2024-11-14T07:26:12,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/74717f8dc4e84fd5b15a8def57d04f9b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/74717f8dc4e84fd5b15a8def57d04f9b 2024-11-14T07:26:12,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/d6bee4c3fe914fd697a57441d109451b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/B/d6bee4c3fe914fd697a57441d109451b 2024-11-14T07:26:12,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/494600e2fe4b4060a3e25d07782e8ad2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/494600e2fe4b4060a3e25d07782e8ad2 2024-11-14T07:26:12,339 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/c0b66e611f0d42e69b502f02e5466905 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/C/c0b66e611f0d42e69b502f02e5466905 2024-11-14T07:26:12,344 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/recovered.edits/508.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3/recovered.edits/508.seqid 2024-11-14T07:26:12,345 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/f7fcaf3f9e5b875de457322a3a36b0e3 2024-11-14T07:26:12,345 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:26:12,348 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,355 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:26:12,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:26:12,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:26:12,360 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569172360"}]},"ts":"9223372036854775807"} 2024-11-14T07:26:12,362 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:26:12,362 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7fcaf3f9e5b875de457322a3a36b0e3, NAME => 'TestAcidGuarantees,,1731569143362.f7fcaf3f9e5b875de457322a3a36b0e3.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:26:12,362 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:26:12,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569172362"}]},"ts":"9223372036854775807"} 2024-11-14T07:26:12,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:26:12,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-14T07:26:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-14T07:26:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:26:12,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:12,428 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 245), OpenFileDescriptor=459 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 518), ProcessCount=11 (was 11), AvailableMemoryMB=5829 (was 5679) - AvailableMemoryMB LEAK? - 2024-11-14T07:26:12,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=5828 2024-11-14T07:26:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:26:12,440 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:26:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:12,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:26:12,442 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:12,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-14T07:26:12,443 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:26:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:12,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742212_1388 (size=959) 2024-11-14T07:26:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:12,856 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:26:12,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742213_1389 (size=53) 2024-11-14T07:26:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:26:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a7fb55e65d21ca143b07686010a18f00, disabling compactions & flushes 2024-11-14T07:26:13,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. after waiting 0 ms 2024-11-14T07:26:13,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7fb55e65d21ca143b07686010a18f00: Waiting for close lock at 1731569173266Disabling compacts and flushes for region at 1731569173266Disabling writes for close at 1731569173267 (+1 ms)Writing region close event to WAL at 1731569173267Closed at 1731569173267 2024-11-14T07:26:13,270 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:26:13,271 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569173270"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569173270"}]},"ts":"1731569173270"} 2024-11-14T07:26:13,274 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:26:13,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:26:13,276 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569173275"}]},"ts":"1731569173275"} 2024-11-14T07:26:13,277 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:26:13,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, ASSIGN}] 2024-11-14T07:26:13,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, ASSIGN 2024-11-14T07:26:13,278 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:26:13,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:13,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, ASSIGN because future has completed 2024-11-14T07:26:13,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:13,596 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,596 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:26:13,597 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,597 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:26:13,597 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,597 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,598 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,599 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName A 2024-11-14T07:26:13,599 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:13,600 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:13,600 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,601 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName B 2024-11-14T07:26:13,601 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:13,602 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:13,602 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,603 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName C 2024-11-14T07:26:13,603 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:13,603 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:13,603 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,604 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,604 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,605 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,605 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,606 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:26:13,607 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,608 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:26:13,609 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened a7fb55e65d21ca143b07686010a18f00; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71286315, jitterRate=0.062248870730400085}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:26:13,609 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:13,609 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for a7fb55e65d21ca143b07686010a18f00: Running coprocessor pre-open hook at 1731569173597Writing region info on filesystem at 1731569173597Initializing all the Stores at 1731569173598 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569173598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569173598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569173598Cleaning up temporary data from old regions at 1731569173605 (+7 ms)Running coprocessor post-open hooks at 1731569173609 (+4 ms)Region opened successfully at 1731569173609 2024-11-14T07:26:13,610 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., pid=97, masterSystemTime=1731569173588 2024-11-14T07:26:13,611 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,612 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:13,612 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:13,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:13,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-14T07:26:13,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 in 180 msec 2024-11-14T07:26:13,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T07:26:13,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, ASSIGN in 339 msec 2024-11-14T07:26:13,619 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:26:13,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569173619"}]},"ts":"1731569173619"} 2024-11-14T07:26:13,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:26:13,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:26:13,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-14T07:26:13,896 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:26:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T07:26:14,605 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:14,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:14,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:14,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:14,607 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f465764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:14,608 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77924115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:14,609 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:14,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:14,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:14,611 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:26:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:26:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20397fe7 2024-11-14T07:26:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:26:14,613 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:26:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:26:14,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:26:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:14,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742214_1390 (size=995) 2024-11-14T07:26:15,026 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T07:26:15,026 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T07:26:15,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:26:15,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE}] 2024-11-14T07:26:15,031 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE 2024-11-14T07:26:15,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE because future has completed 2024-11-14T07:26:15,033 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:26:15,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:15,186 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,186 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:26:15,186 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing a7fb55e65d21ca143b07686010a18f00, disabling compactions & flushes 2024-11-14T07:26:15,186 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,186 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,186 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. after waiting 0 ms 2024-11-14T07:26:15,186 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,190 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:26:15,190 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,190 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for a7fb55e65d21ca143b07686010a18f00: Waiting for close lock at 1731569175186Running coprocessor pre-close hooks at 1731569175186Disabling compacts and flushes for region at 1731569175186Disabling writes for close at 1731569175186Writing region close event to WAL at 1731569175187 (+1 ms)Running coprocessor post-close hooks at 1731569175190 (+3 ms)Closed at 1731569175190 2024-11-14T07:26:15,190 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: a7fb55e65d21ca143b07686010a18f00 to self. 2024-11-14T07:26:15,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=CLOSED 2024-11-14T07:26:15,192 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:15,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-14T07:26:15,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 in 161 msec 2024-11-14T07:26:15,197 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=true 2024-11-14T07:26:15,347 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE because future has completed 2024-11-14T07:26:15,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:15,512 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:26:15,512 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,513 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:26:15,513 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,513 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,515 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,516 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName A 2024-11-14T07:26:15,517 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:15,518 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:15,518 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,519 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName B 2024-11-14T07:26:15,519 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:15,520 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:15,520 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,520 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb55e65d21ca143b07686010a18f00 columnFamilyName C 2024-11-14T07:26:15,520 DEBUG [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:15,521 INFO [StoreOpener-a7fb55e65d21ca143b07686010a18f00-1 {}] regionserver.HStore(327): Store=a7fb55e65d21ca143b07686010a18f00/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:15,521 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,522 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,523 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,524 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,524 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,525 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:26:15,526 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,527 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened a7fb55e65d21ca143b07686010a18f00; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72053868, jitterRate=0.07368630170822144}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:26:15,527 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,528 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for a7fb55e65d21ca143b07686010a18f00: Running coprocessor pre-open hook at 1731569175513Writing region info on filesystem at 1731569175513Initializing all the Stores at 1731569175514 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569175514Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569175515 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569175515Cleaning up temporary data from old regions at 1731569175524 (+9 ms)Running coprocessor post-open hooks at 1731569175527 (+3 ms)Region opened successfully at 1731569175528 (+1 ms) 2024-11-14T07:26:15,528 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., pid=102, masterSystemTime=1731569175505 2024-11-14T07:26:15,530 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,530 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,530 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:15,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-14T07:26:15,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 in 181 msec 2024-11-14T07:26:15,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T07:26:15,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, REOPEN/MOVE in 503 msec 2024-11-14T07:26:15,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-14T07:26:15,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-14T07:26:15,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-14T07:26:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-14T07:26:15,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:15,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,540 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea6e69d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,542 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd8484c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,543 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,544 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5287698b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d32ec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777adac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@988da3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef6a253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66706f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600c2100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa5fc2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618d06bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b1948e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337d2835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4c166a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bab261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2bbab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166ebb22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,570 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e49ad43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:15,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:15,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27daa15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:15,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0c3bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:15,576 DEBUG [Thread-1763 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,576 DEBUG [Thread-1748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:15,576 DEBUG [Thread-1765 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,576 DEBUG [Thread-1761 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,577 DEBUG [Thread-1759 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T07:26:15,578 DEBUG [Thread-1763 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,578 DEBUG [Thread-1763 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,578 DEBUG [Thread-1767 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,579 DEBUG [Thread-1756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:15,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,581 DEBUG [Thread-1765 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,581 DEBUG [Thread-1765 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,581 DEBUG [Thread-1750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,581 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:15,582 DEBUG [Thread-1754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,582 DEBUG [Thread-1752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,583 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:15,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:26:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:15,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:15,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:15,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db8c4eed5d4941cd859a3a6ad53fbca8_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569175592/Put/seqid=0 2024-11-14T07:26:15,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742215_1391 (size=17034) 2024-11-14T07:26:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569235625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569235626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569235627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569235629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569235633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:15,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569235744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569235744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569235744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569235744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569235745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:15,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569235954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569235954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569235954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569235955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569235955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,033 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:16,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114db8c4eed5d4941cd859a3a6ad53fbca8_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db8c4eed5d4941cd859a3a6ad53fbca8_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:16,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/5bed58c6ec6745d2bd45b8ca08e9cf1f, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:16,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/5bed58c6ec6745d2bd45b8ca08e9cf1f is 175, key is test_row_0/A:col10/1731569175592/Put/seqid=0 2024-11-14T07:26:16,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742216_1392 (size=48139) 2024-11-14T07:26:16,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,042 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/5bed58c6ec6745d2bd45b8ca08e9cf1f 2024-11-14T07:26:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/dfdbac8af5b24b46853099c88b3d28a5 is 50, key is test_row_0/B:col10/1731569175592/Put/seqid=0 2024-11-14T07:26:16,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742217_1393 (size=12001) 2024-11-14T07:26:16,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569236264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569236264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569236264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569236264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569236264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/dfdbac8af5b24b46853099c88b3d28a5 2024-11-14T07:26:16,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/8158afa0316048c687a52cc8553ea974 is 50, key is test_row_0/C:col10/1731569175592/Put/seqid=0 2024-11-14T07:26:16,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742218_1394 (size=12001) 2024-11-14T07:26:16,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:16,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569236774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569236774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569236774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569236775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569236785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:16,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:16,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/8158afa0316048c687a52cc8553ea974 2024-11-14T07:26:16,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/5bed58c6ec6745d2bd45b8ca08e9cf1f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f 2024-11-14T07:26:16,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f, entries=250, sequenceid=16, filesize=47.0 K 2024-11-14T07:26:16,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/dfdbac8af5b24b46853099c88b3d28a5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5 2024-11-14T07:26:16,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T07:26:16,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/8158afa0316048c687a52cc8553ea974 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974 2024-11-14T07:26:16,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T07:26:16,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a7fb55e65d21ca143b07686010a18f00 in 1310ms, sequenceid=16, compaction requested=false 2024-11-14T07:26:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T07:26:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:16,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T07:26:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:26:16,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f8dd851c414a48e9b4a1915c4103dae5_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569175626/Put/seqid=0 2024-11-14T07:26:16,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742219_1395 (size=12154) 2024-11-14T07:26:17,200 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:26:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:17,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f8dd851c414a48e9b4a1915c4103dae5_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8dd851c414a48e9b4a1915c4103dae5_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/80706b294c114a43a3db7ab5ad678523, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/80706b294c114a43a3db7ab5ad678523 is 175, key is test_row_0/A:col10/1731569175626/Put/seqid=0 2024-11-14T07:26:17,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742220_1396 (size=30955) 2024-11-14T07:26:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/80706b294c114a43a3db7ab5ad678523 2024-11-14T07:26:17,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b830743519044f649c11a2a73a9c5c7c is 50, key is test_row_0/B:col10/1731569175626/Put/seqid=0 2024-11-14T07:26:17,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742221_1397 (size=12001) 2024-11-14T07:26:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:17,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:17,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b830743519044f649c11a2a73a9c5c7c 2024-11-14T07:26:17,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/19e97061f6764141b1b25277d1193dad is 50, key is test_row_0/C:col10/1731569175626/Put/seqid=0 2024-11-14T07:26:17,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742222_1398 (size=12001) 2024-11-14T07:26:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569237798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569237799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569237806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569237808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569237808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569237915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569237915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569237915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569237924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569237924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569238125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569238125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569238125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569238134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569238134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,202 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/19e97061f6764141b1b25277d1193dad 2024-11-14T07:26:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/80706b294c114a43a3db7ab5ad678523 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523 2024-11-14T07:26:18,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523, entries=150, sequenceid=40, filesize=30.2 K 2024-11-14T07:26:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b830743519044f649c11a2a73a9c5c7c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c 2024-11-14T07:26:18,214 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T07:26:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/19e97061f6764141b1b25277d1193dad as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad 2024-11-14T07:26:18,219 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T07:26:18,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a7fb55e65d21ca143b07686010a18f00 in 1262ms, sequenceid=40, compaction requested=false 2024-11-14T07:26:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T07:26:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T07:26:18,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T07:26:18,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-14T07:26:18,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6480 sec 2024-11-14T07:26:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:18,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:18,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:26:18,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b38ce2be73f0488aabea0c338a91b2b3_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:18,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742223_1399 (size=14594) 2024-11-14T07:26:18,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:18,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b38ce2be73f0488aabea0c338a91b2b3_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b38ce2be73f0488aabea0c338a91b2b3_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:18,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3b2739e526d5491a801152411640e0e7, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:18,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3b2739e526d5491a801152411640e0e7 is 175, key is test_row_0/A:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:18,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742224_1400 (size=39549) 2024-11-14T07:26:18,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569238469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569238473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569238473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569238474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569238475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569238584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569238584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569238584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569238585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569238595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569238794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569238794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569238794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569238795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569238804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:18,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3b2739e526d5491a801152411640e0e7 2024-11-14T07:26:18,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/79cb14920f424acfa6decadea38e22a7 is 50, key is test_row_0/B:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742225_1401 (size=12001) 2024-11-14T07:26:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569239104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569239104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569239104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569239104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569239115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/79cb14920f424acfa6decadea38e22a7 2024-11-14T07:26:19,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/897fc1f419494684b4e9f540788696aa is 50, key is test_row_0/C:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:19,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742226_1402 (size=12001) 2024-11-14T07:26:19,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569239614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569239614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569239614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569239625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569239625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:19,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/897fc1f419494684b4e9f540788696aa 2024-11-14T07:26:19,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3b2739e526d5491a801152411640e0e7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7 2024-11-14T07:26:19,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7, entries=200, sequenceid=54, filesize=38.6 K 2024-11-14T07:26:19,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/79cb14920f424acfa6decadea38e22a7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7 2024-11-14T07:26:19,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T07:26:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/897fc1f419494684b4e9f540788696aa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa 2024-11-14T07:26:19,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T07:26:19,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a7fb55e65d21ca143b07686010a18f00 in 1254ms, sequenceid=54, compaction requested=true 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:19,694 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:19,694 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:19,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:19,695 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:19,695 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:19,695 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=115.9 K 2024-11-14T07:26:19,695 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.2 K 2024-11-14T07:26:19,695 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7] 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting dfdbac8af5b24b46853099c88b3d28a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731569175592 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bed58c6ec6745d2bd45b8ca08e9cf1f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731569175590 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b830743519044f649c11a2a73a9c5c7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569175615 2024-11-14T07:26:19,695 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80706b294c114a43a3db7ab5ad678523, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569175615 2024-11-14T07:26:19,696 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 79cb14920f424acfa6decadea38e22a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:19,696 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b2739e526d5491a801152411640e0e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:19,704 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:19,705 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/9456a6c2f15241ddb34c7d8564021bc7 is 50, key is test_row_0/B:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:19,706 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:19,709 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114045eb3054ec342529ecb86b4f35c6a2a_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:19,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742227_1403 (size=12104) 2024-11-14T07:26:19,711 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114045eb3054ec342529ecb86b4f35c6a2a_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:19,711 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114045eb3054ec342529ecb86b4f35c6a2a_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:19,715 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/9456a6c2f15241ddb34c7d8564021bc7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9456a6c2f15241ddb34c7d8564021bc7 2024-11-14T07:26:19,719 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into 9456a6c2f15241ddb34c7d8564021bc7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:19,719 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:19,719 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=13, startTime=1731569179694; duration=0sec 2024-11-14T07:26:19,719 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:19,719 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:19,719 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:19,720 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:19,720 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:19,720 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:19,720 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.2 K 2024-11-14T07:26:19,720 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8158afa0316048c687a52cc8553ea974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731569175592 2024-11-14T07:26:19,720 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e97061f6764141b1b25277d1193dad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731569175615 2024-11-14T07:26:19,722 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 897fc1f419494684b4e9f540788696aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:19,730 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:19,731 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/102684a4a87748d7972691e375604480 is 50, key is test_row_0/C:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:19,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742228_1404 (size=4469) 2024-11-14T07:26:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742229_1405 (size=12104) 2024-11-14T07:26:19,744 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#348 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:19,744 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/de86356b27024d61af95a1a33dd21c9b is 175, key is test_row_0/A:col10/1731569178439/Put/seqid=0 2024-11-14T07:26:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T07:26:19,745 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:19,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T07:26:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:26:19,747 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:19,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:19,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:19,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742230_1406 (size=31058) 2024-11-14T07:26:19,756 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/de86356b27024d61af95a1a33dd21c9b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b 2024-11-14T07:26:19,761 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into de86356b27024d61af95a1a33dd21c9b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:19,761 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:19,761 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=13, startTime=1731569179694; duration=0sec 2024-11-14T07:26:19,761 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:19,761 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:26:19,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T07:26:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:19,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T07:26:19,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111401dc4d6ff25d4d8ab66544392d9b18e2_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569178474/Put/seqid=0 2024-11-14T07:26:19,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742231_1407 (size=12154) 2024-11-14T07:26:19,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:19,919 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111401dc4d6ff25d4d8ab66544392d9b18e2_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401dc4d6ff25d4d8ab66544392d9b18e2_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:19,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3df1960596f642b4b5421e4e55fd536e, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3df1960596f642b4b5421e4e55fd536e is 175, key is test_row_0/A:col10/1731569178474/Put/seqid=0 2024-11-14T07:26:19,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742232_1408 (size=30955) 2024-11-14T07:26:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:26:20,147 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/102684a4a87748d7972691e375604480 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/102684a4a87748d7972691e375604480 2024-11-14T07:26:20,151 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into 102684a4a87748d7972691e375604480(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:20,151 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:20,152 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=13, startTime=1731569179694; duration=0sec 2024-11-14T07:26:20,152 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:20,152 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:20,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3df1960596f642b4b5421e4e55fd536e 2024-11-14T07:26:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/13bd2208ee7e425aa12570747cfe8b1d is 50, key is test_row_0/B:col10/1731569178474/Put/seqid=0 2024-11-14T07:26:20,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742233_1409 (size=12001) 2024-11-14T07:26:20,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/13bd2208ee7e425aa12570747cfe8b1d 2024-11-14T07:26:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/79b8fdcb241f462c9959cebc09850157 is 50, key is test_row_0/C:col10/1731569178474/Put/seqid=0 2024-11-14T07:26:20,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742234_1410 (size=12001) 2024-11-14T07:26:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:26:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:20,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569240644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569240646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,654 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569240651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569240652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569240654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/79b8fdcb241f462c9959cebc09850157 2024-11-14T07:26:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3df1960596f642b4b5421e4e55fd536e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e 2024-11-14T07:26:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569240765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569240764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569240765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569240765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,773 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e, entries=150, sequenceid=78, filesize=30.2 K 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/13bd2208ee7e425aa12570747cfe8b1d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d 2024-11-14T07:26:20,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T07:26:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/79b8fdcb241f462c9959cebc09850157 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157 2024-11-14T07:26:20,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157, entries=150, sequenceid=78, filesize=11.7 K 2024-11-14T07:26:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a7fb55e65d21ca143b07686010a18f00 in 882ms, sequenceid=78, compaction requested=false 2024-11-14T07:26:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T07:26:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T07:26:20,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T07:26:20,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-11-14T07:26:20,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0410 sec 2024-11-14T07:26:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T07:26:20,885 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:20,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-14T07:26:20,887 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:20,887 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:20,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:20,987 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:20,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T07:26:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:20,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3dcb149745d49aba47886c881c3a03b_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:21,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742235_1411 (size=17034) 2024-11-14T07:26:21,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569241022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569241023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569241028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569241032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569241134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569241144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569241145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569241145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:21,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569241344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569241354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569241354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569241354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:21,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3dcb149745d49aba47886c881c3a03b_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dcb149745d49aba47886c881c3a03b_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:21,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/4b1443ff4bbf4bd58651e4cf9469f3c8, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:21,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/4b1443ff4bbf4bd58651e4cf9469f3c8 is 175, key is test_row_0/A:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:21,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742236_1412 (size=48139) 2024-11-14T07:26:21,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:21,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569241654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569241665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569241665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569241674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:21,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/4b1443ff4bbf4bd58651e4cf9469f3c8 2024-11-14T07:26:21,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b67ddf720a52421fadd78e372dc0fc93 is 50, key is test_row_0/B:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:21,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742237_1413 (size=12001) 2024-11-14T07:26:21,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:21,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:22,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569242175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569242185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569242185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569242185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:22,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b67ddf720a52421fadd78e372dc0fc93 2024-11-14T07:26:22,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/d4ac486ae2624bcba66efaaa2982042f is 50, key is test_row_0/C:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:22,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742238_1414 (size=12001) 2024-11-14T07:26:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:22,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/d4ac486ae2624bcba66efaaa2982042f 2024-11-14T07:26:22,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/4b1443ff4bbf4bd58651e4cf9469f3c8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8 2024-11-14T07:26:22,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8, entries=250, sequenceid=95, filesize=47.0 K 2024-11-14T07:26:22,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/b67ddf720a52421fadd78e372dc0fc93 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93 2024-11-14T07:26:22,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T07:26:22,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/d4ac486ae2624bcba66efaaa2982042f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f 2024-11-14T07:26:22,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T07:26:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a7fb55e65d21ca143b07686010a18f00 in 1318ms, sequenceid=95, compaction requested=true 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:22,305 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:22,305 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:22,306 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:22,306 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:22,307 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,307 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9456a6c2f15241ddb34c7d8564021bc7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.3 K 2024-11-14T07:26:22,307 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:22,307 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:22,307 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,307 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=107.6 K 2024-11-14T07:26:22,307 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,307 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8] 2024-11-14T07:26:22,307 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9456a6c2f15241ddb34c7d8564021bc7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:22,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting de86356b27024d61af95a1a33dd21c9b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:22,308 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bd2208ee7e425aa12570747cfe8b1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731569178472 2024-11-14T07:26:22,308 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b67ddf720a52421fadd78e372dc0fc93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:22,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3df1960596f642b4b5421e4e55fd536e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731569178472 2024-11-14T07:26:22,309 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1443ff4bbf4bd58651e4cf9469f3c8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:22,329 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:22,334 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:22,334 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/fd267f2748fe42b9bb1e71d1c8fffd32 is 50, key is test_row_0/B:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:22,337 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111477f4515e193649d4bd4c5d33d2534baf_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:22,339 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111477f4515e193649d4bd4c5d33d2534baf_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:22,339 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111477f4515e193649d4bd4c5d33d2534baf_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:22,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742239_1415 (size=12207) 2024-11-14T07:26:22,387 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/fd267f2748fe42b9bb1e71d1c8fffd32 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fd267f2748fe42b9bb1e71d1c8fffd32 2024-11-14T07:26:22,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742240_1416 (size=4469) 2024-11-14T07:26:22,394 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#356 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:22,394 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/8beb009124be455a914d21e631967e19 is 175, key is test_row_0/A:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:22,396 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into fd267f2748fe42b9bb1e71d1c8fffd32(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:22,396 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:22,396 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=13, startTime=1731569182305; duration=0sec 2024-11-14T07:26:22,396 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:22,396 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:22,396 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:22,397 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:22,397 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:22,397 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,397 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/102684a4a87748d7972691e375604480, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.3 K 2024-11-14T07:26:22,398 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 102684a4a87748d7972691e375604480, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569177788 2024-11-14T07:26:22,398 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b8fdcb241f462c9959cebc09850157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731569178472 2024-11-14T07:26:22,398 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ac486ae2624bcba66efaaa2982042f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742241_1417 (size=31161) 2024-11-14T07:26:22,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-14T07:26:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:22,426 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:22,426 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/c53ba964cbad4500b3d745d53d89f669 is 50, key is test_row_0/C:col10/1731569180986/Put/seqid=0 2024-11-14T07:26:22,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T07:26:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742242_1418 (size=12207) 2024-11-14T07:26:22,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140b37cfd87c9e4da7987e2ac353aa109f_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569181011/Put/seqid=0 2024-11-14T07:26:22,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742243_1419 (size=12154) 2024-11-14T07:26:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:22,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140b37cfd87c9e4da7987e2ac353aa109f_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b37cfd87c9e4da7987e2ac353aa109f_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/46a003e63bb3458aa02c33fa967f9459, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:22,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/46a003e63bb3458aa02c33fa967f9459 is 175, key is test_row_0/A:col10/1731569181011/Put/seqid=0 2024-11-14T07:26:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742244_1420 (size=30955) 2024-11-14T07:26:22,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:22,827 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/8beb009124be455a914d21e631967e19 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19 2024-11-14T07:26:22,832 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into 8beb009124be455a914d21e631967e19(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:22,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:22,832 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=13, startTime=1731569182305; duration=0sec 2024-11-14T07:26:22,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:22,832 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:22,878 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/c53ba964cbad4500b3d745d53d89f669 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/c53ba964cbad4500b3d745d53d89f669 2024-11-14T07:26:22,884 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into c53ba964cbad4500b3d745d53d89f669(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:22,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:22,884 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=13, startTime=1731569182305; duration=0sec 2024-11-14T07:26:22,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:22,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:22,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569242871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:22,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/46a003e63bb3458aa02c33fa967f9459 2024-11-14T07:26:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/bca7a0550ca349aab49d10b334dbf41e is 50, key is test_row_0/B:col10/1731569181011/Put/seqid=0 2024-11-14T07:26:22,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742245_1421 (size=12001) 2024-11-14T07:26:22,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/bca7a0550ca349aab49d10b334dbf41e 2024-11-14T07:26:23,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569242995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/1e917d0128cc41e696a32242bdd5c2c7 is 50, key is test_row_0/C:col10/1731569181011/Put/seqid=0 2024-11-14T07:26:23,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742246_1422 (size=12001) 2024-11-14T07:26:23,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/1e917d0128cc41e696a32242bdd5c2c7 2024-11-14T07:26:23,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/46a003e63bb3458aa02c33fa967f9459 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459 2024-11-14T07:26:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:23,037 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459, entries=150, sequenceid=116, filesize=30.2 K 2024-11-14T07:26:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/bca7a0550ca349aab49d10b334dbf41e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e 2024-11-14T07:26:23,047 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T07:26:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/1e917d0128cc41e696a32242bdd5c2c7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7 2024-11-14T07:26:23,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T07:26:23,058 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a7fb55e65d21ca143b07686010a18f00 in 631ms, sequenceid=116, compaction requested=false 2024-11-14T07:26:23,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:23,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:23,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-14T07:26:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-14T07:26:23,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T07:26:23,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-14T07:26:23,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1800 sec 2024-11-14T07:26:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:23,198 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:23,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T07:26:23,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c6393921b74840f58b0d7b5050eadab4_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:23,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742247_1423 (size=14744) 2024-11-14T07:26:23,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569243241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569243248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569243253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569243253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569243258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569243365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569243375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569243375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569243375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569243376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569243584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569243595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569243595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569243595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569243595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:23,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c6393921b74840f58b0d7b5050eadab4_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c6393921b74840f58b0d7b5050eadab4_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:23,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9f2f877c2f2d4ffb929d2741e464aaa7, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:23,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9f2f877c2f2d4ffb929d2741e464aaa7 is 175, key is test_row_0/A:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:23,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742248_1424 (size=39699) 2024-11-14T07:26:23,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569243895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569243914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569243914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569243915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569243915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9f2f877c2f2d4ffb929d2741e464aaa7 2024-11-14T07:26:24,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7991501379a84c4e99b9f4d3c79f8dd8 is 50, key is test_row_0/B:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:24,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742249_1425 (size=12101) 2024-11-14T07:26:24,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7991501379a84c4e99b9f4d3c79f8dd8 2024-11-14T07:26:24,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/dd45615773024ccd9eec2b1f1bcc8937 is 50, key is test_row_0/C:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742250_1426 (size=12101) 2024-11-14T07:26:24,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/dd45615773024ccd9eec2b1f1bcc8937 2024-11-14T07:26:24,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9f2f877c2f2d4ffb929d2741e464aaa7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7 2024-11-14T07:26:24,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7, entries=200, sequenceid=135, filesize=38.8 K 2024-11-14T07:26:24,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7991501379a84c4e99b9f4d3c79f8dd8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8 2024-11-14T07:26:24,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8, entries=150, sequenceid=135, filesize=11.8 K 2024-11-14T07:26:24,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/dd45615773024ccd9eec2b1f1bcc8937 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937 2024-11-14T07:26:24,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937, entries=150, sequenceid=135, filesize=11.8 K 2024-11-14T07:26:24,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for a7fb55e65d21ca143b07686010a18f00 in 947ms, sequenceid=135, compaction requested=true 2024-11-14T07:26:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:24,146 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:24,146 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:24,147 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:24,147 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:24,147 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:24,147 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=99.4 K 2024-11-14T07:26:24,147 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:24,147 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7] 2024-11-14T07:26:24,148 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8beb009124be455a914d21e631967e19, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:24,149 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a003e63bb3458aa02c33fa967f9459, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731569181011 2024-11-14T07:26:24,149 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:24,149 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:24,149 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:24,149 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fd267f2748fe42b9bb1e71d1c8fffd32, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.5 K 2024-11-14T07:26:24,149 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2f877c2f2d4ffb929d2741e464aaa7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:24,150 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fd267f2748fe42b9bb1e71d1c8fffd32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:24,151 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting bca7a0550ca349aab49d10b334dbf41e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731569181011 2024-11-14T07:26:24,153 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7991501379a84c4e99b9f4d3c79f8dd8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:24,190 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:24,197 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:24,197 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142fea2f25f42d46b7a90ef8afcc8e5f59_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:24,197 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/465eb78fde8e4bf88205bac0e05c22ff is 50, key is test_row_0/B:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:24,199 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142fea2f25f42d46b7a90ef8afcc8e5f59_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:24,199 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142fea2f25f42d46b7a90ef8afcc8e5f59_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742251_1427 (size=12409) 2024-11-14T07:26:24,206 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/465eb78fde8e4bf88205bac0e05c22ff as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/465eb78fde8e4bf88205bac0e05c22ff 2024-11-14T07:26:24,209 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into 465eb78fde8e4bf88205bac0e05c22ff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:24,210 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:24,210 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=13, startTime=1731569184146; duration=0sec 2024-11-14T07:26:24,210 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:24,210 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:24,210 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:24,216 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:24,216 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:24,216 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:24,216 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/c53ba964cbad4500b3d745d53d89f669, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=35.5 K 2024-11-14T07:26:24,221 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting c53ba964cbad4500b3d745d53d89f669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731569180650 2024-11-14T07:26:24,221 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e917d0128cc41e696a32242bdd5c2c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731569181011 2024-11-14T07:26:24,222 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting dd45615773024ccd9eec2b1f1bcc8937, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:24,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742252_1428 (size=4469) 2024-11-14T07:26:24,225 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#365 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:24,225 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/75eea87d6bec4a6281a74801e233cb85 is 175, key is test_row_0/A:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:24,234 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:24,234 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/1d4617bf40b948458ac1475456ef0a88 is 50, key is test_row_0/C:col10/1731569182855/Put/seqid=0 2024-11-14T07:26:24,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742253_1429 (size=31363) 2024-11-14T07:26:24,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742254_1430 (size=12409) 2024-11-14T07:26:24,243 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/75eea87d6bec4a6281a74801e233cb85 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85 2024-11-14T07:26:24,263 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into 75eea87d6bec4a6281a74801e233cb85(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:24,263 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:24,263 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=13, startTime=1731569184146; duration=0sec 2024-11-14T07:26:24,264 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:24,264 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:24,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:24,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T07:26:24,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111462370c2f986841248f27a6681b1fb7de_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569183250/Put/seqid=0 2024-11-14T07:26:24,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742255_1431 (size=14794) 2024-11-14T07:26:24,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569244455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569244455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569244456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569244457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569244458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569244575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569244575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569244575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569244575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569244576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,645 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/1d4617bf40b948458ac1475456ef0a88 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1d4617bf40b948458ac1475456ef0a88 2024-11-14T07:26:24,652 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into 1d4617bf40b948458ac1475456ef0a88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:24,652 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:24,652 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=13, startTime=1731569184146; duration=0sec 2024-11-14T07:26:24,652 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:24,652 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569244785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569244785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569244785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569244785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569244796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:24,843 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:24,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111462370c2f986841248f27a6681b1fb7de_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462370c2f986841248f27a6681b1fb7de_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:24,847 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3f306226ecc842d586e54c30fd14b1ba, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:24,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3f306226ecc842d586e54c30fd14b1ba is 175, key is test_row_0/A:col10/1731569183250/Put/seqid=0 2024-11-14T07:26:24,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742256_1432 (size=39749) 2024-11-14T07:26:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T07:26:25,046 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:25,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-14T07:26:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:25,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:25,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:25,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:25,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569245104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569245104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569245104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569245104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569245115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:25,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3f306226ecc842d586e54c30fd14b1ba 2024-11-14T07:26:25,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/80f1530771bd403f9aabcc056c3416e6 is 50, key is test_row_0/B:col10/1731569183250/Put/seqid=0 2024-11-14T07:26:25,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:25,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742257_1433 (size=12151) 2024-11-14T07:26:25,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569245625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569245625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569245625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569245625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569245634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:25,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:25,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:25,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/80f1530771bd403f9aabcc056c3416e6 2024-11-14T07:26:25,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/0f5a4c4f47bd4d848a57519943f9ebcd is 50, key is test_row_0/C:col10/1731569183250/Put/seqid=0 2024-11-14T07:26:25,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:25,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742258_1434 (size=12151) 2024-11-14T07:26:25,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/0f5a4c4f47bd4d848a57519943f9ebcd 2024-11-14T07:26:25,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/3f306226ecc842d586e54c30fd14b1ba as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba 2024-11-14T07:26:25,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba, entries=200, sequenceid=158, filesize=38.8 K 2024-11-14T07:26:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/80f1530771bd403f9aabcc056c3416e6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6 2024-11-14T07:26:25,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T07:26:25,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/0f5a4c4f47bd4d848a57519943f9ebcd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd 2024-11-14T07:26:25,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T07:26:25,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a7fb55e65d21ca143b07686010a18f00 in 1463ms, sequenceid=158, compaction requested=false 2024-11-14T07:26:25,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:25,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-14T07:26:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:25,980 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T07:26:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f34575229c0a4eb48b75492c38361101_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569184454/Put/seqid=0 2024-11-14T07:26:26,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742259_1435 (size=12304) 2024-11-14T07:26:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f34575229c0a4eb48b75492c38361101_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f34575229c0a4eb48b75492c38361101_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/79fd7ac48941456aae7f9159b42db8fa, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/79fd7ac48941456aae7f9159b42db8fa is 175, key is test_row_0/A:col10/1731569184454/Put/seqid=0 2024-11-14T07:26:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742260_1436 (size=31105) 2024-11-14T07:26:26,059 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/79fd7ac48941456aae7f9159b42db8fa 2024-11-14T07:26:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/fe6b2b1ca8b04ea487fe4416d5062038 is 50, key is test_row_0/B:col10/1731569184454/Put/seqid=0 2024-11-14T07:26:26,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742261_1437 (size=12151) 2024-11-14T07:26:26,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/fe6b2b1ca8b04ea487fe4416d5062038 2024-11-14T07:26:26,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/9d13947f49174ef3ad9bc648aa2a1aa5 is 50, key is test_row_0/C:col10/1731569184454/Put/seqid=0 2024-11-14T07:26:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:26,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742262_1438 (size=12151) 2024-11-14T07:26:26,216 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/9d13947f49174ef3ad9bc648aa2a1aa5 2024-11-14T07:26:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/79fd7ac48941456aae7f9159b42db8fa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa 2024-11-14T07:26:26,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa, entries=150, sequenceid=174, filesize=30.4 K 2024-11-14T07:26:26,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/fe6b2b1ca8b04ea487fe4416d5062038 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038 2024-11-14T07:26:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,231 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T07:26:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/9d13947f49174ef3ad9bc648aa2a1aa5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5 2024-11-14T07:26:26,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T07:26:26,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for a7fb55e65d21ca143b07686010a18f00 in 264ms, sequenceid=174, compaction requested=true 2024-11-14T07:26:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-14T07:26:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-14T07:26:26,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T07:26:26,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-14T07:26:26,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.2010 sec 2024-11-14T07:26:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fabfd0aba36747719c2c39862a780cce_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742264_1440 (size=24758) 2024-11-14T07:26:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fabfd0aba36747719c2c39862a780cce_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fabfd0aba36747719c2c39862a780cce_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:26,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/c6b583ad22c34f00a7df690c011de3f3, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:26,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/c6b583ad22c34f00a7df690c011de3f3 is 175, key is test_row_0/A:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742263_1439 (size=74395) 2024-11-14T07:26:26,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/c6b583ad22c34f00a7df690c011de3f3 2024-11-14T07:26:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/158ffb53db1a4fb0a22b81b9f3602208 is 50, key is test_row_0/B:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:26,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742265_1441 (size=24105) 2024-11-14T07:26:26,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/158ffb53db1a4fb0a22b81b9f3602208 2024-11-14T07:26:26,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/b7f1a8ff0f00420e89ec2bed25ce329c is 50, key is test_row_0/C:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569246985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569246986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569246987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569247001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569247002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742266_1442 (size=12151) 2024-11-14T07:26:27,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/b7f1a8ff0f00420e89ec2bed25ce329c 2024-11-14T07:26:27,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/c6b583ad22c34f00a7df690c011de3f3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3 2024-11-14T07:26:27,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3, entries=400, sequenceid=185, filesize=72.7 K 2024-11-14T07:26:27,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/158ffb53db1a4fb0a22b81b9f3602208 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208 2024-11-14T07:26:27,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208, entries=400, sequenceid=185, filesize=23.5 K 2024-11-14T07:26:27,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/b7f1a8ff0f00420e89ec2bed25ce329c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c 2024-11-14T07:26:27,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c, entries=150, sequenceid=185, filesize=11.9 K 2024-11-14T07:26:27,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a7fb55e65d21ca143b07686010a18f00 in 213ms, sequenceid=185, compaction requested=true 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:27,055 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:27,055 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:27,062 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:27,062 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:27,062 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60816 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:27,062 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,062 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:27,062 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,063 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=172.5 K 2024-11-14T07:26:27,063 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/465eb78fde8e4bf88205bac0e05c22ff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=59.4 K 2024-11-14T07:26:27,063 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,063 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3] 2024-11-14T07:26:27,063 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 465eb78fde8e4bf88205bac0e05c22ff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75eea87d6bec4a6281a74801e233cb85, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 80f1530771bd403f9aabcc056c3416e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731569183248 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f306226ecc842d586e54c30fd14b1ba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731569183248 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fe6b2b1ca8b04ea487fe4416d5062038, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731569184454 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79fd7ac48941456aae7f9159b42db8fa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731569184454 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 158ffb53db1a4fb0a22b81b9f3602208, keycount=400, bloomtype=ROW, size=23.5 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186776 2024-11-14T07:26:27,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b583ad22c34f00a7df690c011de3f3, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186776 2024-11-14T07:26:27,086 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:27,103 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#378 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:27,103 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/93ef9be7c1044ffd9df9bef7c891481d is 50, key is test_row_0/B:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:27,115 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149297873d21b94d1a80f7e70b9aac5987_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:27,126 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149297873d21b94d1a80f7e70b9aac5987_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:27,126 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149297873d21b94d1a80f7e70b9aac5987_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:27,146 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:27,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T07:26:27,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742267_1443 (size=12595) 2024-11-14T07:26:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569247154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569247157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742268_1444 (size=4469) 2024-11-14T07:26:27,187 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#377 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:27,188 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/0f1a0aa486d544a4b6a64976200fa83a is 175, key is test_row_0/A:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T07:26:27,199 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:27,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-14T07:26:27,202 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:27,203 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:27,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569247171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569247175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569247174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f8a1842f56b4c0181e54e3b870c28cf_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569187000/Put/seqid=0 2024-11-14T07:26:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742269_1445 (size=31549) 2024-11-14T07:26:27,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742270_1446 (size=12304) 2024-11-14T07:26:27,248 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/0f1a0aa486d544a4b6a64976200fa83a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a 2024-11-14T07:26:27,256 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into 0f1a0aa486d544a4b6a64976200fa83a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:27,256 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:27,256 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=12, startTime=1731569187055; duration=0sec 2024-11-14T07:26:27,257 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:27,257 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:27,257 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:27,258 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:27,258 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:27,258 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,258 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1d4617bf40b948458ac1475456ef0a88, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=47.7 K 2024-11-14T07:26:27,259 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4617bf40b948458ac1475456ef0a88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731569182803 2024-11-14T07:26:27,259 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5a4c4f47bd4d848a57519943f9ebcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731569183248 2024-11-14T07:26:27,259 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d13947f49174ef3ad9bc648aa2a1aa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731569184454 2024-11-14T07:26:27,260 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f1a8ff0f00420e89ec2bed25ce329c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186785 2024-11-14T07:26:27,275 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:27,276 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/02ad770ce1294e5aa1e739677f3d7c30 is 50, key is test_row_0/C:col10/1731569186832/Put/seqid=0 2024-11-14T07:26:27,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742271_1447 (size=12595) 2024-11-14T07:26:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569247284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569247284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,296 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/02ad770ce1294e5aa1e739677f3d7c30 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/02ad770ce1294e5aa1e739677f3d7c30 2024-11-14T07:26:27,305 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into 02ad770ce1294e5aa1e739677f3d7c30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:27,305 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:27,305 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=12, startTime=1731569187055; duration=0sec 2024-11-14T07:26:27,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:27,308 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569247316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569247316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569247318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:27,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:27,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569247505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569247505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569247534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569247534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569247545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,579 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/93ef9be7c1044ffd9df9bef7c891481d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/93ef9be7c1044ffd9df9bef7c891481d 2024-11-14T07:26:27,592 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into 93ef9be7c1044ffd9df9bef7c891481d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:27,592 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:27,593 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=12, startTime=1731569187055; duration=0sec 2024-11-14T07:26:27,593 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:27,593 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:27,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:27,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f8a1842f56b4c0181e54e3b870c28cf_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f8a1842f56b4c0181e54e3b870c28cf_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:27,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7986d0e9bdee4a3aa9fb431e4a244ac0, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:27,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7986d0e9bdee4a3aa9fb431e4a244ac0 is 175, key is test_row_0/A:col10/1731569187000/Put/seqid=0 2024-11-14T07:26:27,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742272_1448 (size=31105) 2024-11-14T07:26:27,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7986d0e9bdee4a3aa9fb431e4a244ac0 2024-11-14T07:26:27,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/0efba5527fad4e878efbe5bbca1eaf13 is 50, key is test_row_0/B:col10/1731569187000/Put/seqid=0 2024-11-14T07:26:27,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742273_1449 (size=12151) 2024-11-14T07:26:27,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/0efba5527fad4e878efbe5bbca1eaf13 2024-11-14T07:26:27,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2d23a498da154b9a8706d994fa3c0977 is 50, key is test_row_0/C:col10/1731569187000/Put/seqid=0 2024-11-14T07:26:27,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:27,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:27,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:27,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569247825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569247825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569247854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569247865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:27,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569247865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:27,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742274_1450 (size=12151) 2024-11-14T07:26:27,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:27,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2d23a498da154b9a8706d994fa3c0977 2024-11-14T07:26:28,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7986d0e9bdee4a3aa9fb431e4a244ac0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0 2024-11-14T07:26:28,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0, entries=150, sequenceid=212, filesize=30.4 K 2024-11-14T07:26:28,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/0efba5527fad4e878efbe5bbca1eaf13 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13 2024-11-14T07:26:28,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T07:26:28,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2d23a498da154b9a8706d994fa3c0977 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977 2024-11-14T07:26:28,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T07:26:28,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a7fb55e65d21ca143b07686010a18f00 in 1161ms, sequenceid=212, compaction requested=false 2024-11-14T07:26:28,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:28,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:28,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:28,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111428e0549ed9df49fc9fbb391aa145c3ef_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:28,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742275_1451 (size=17284) 2024-11-14T07:26:28,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:28,424 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111428e0549ed9df49fc9fbb391aa145c3ef_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111428e0549ed9df49fc9fbb391aa145c3ef_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:28,426 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/28eba26a491e435199baac86323da90c, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:28,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/28eba26a491e435199baac86323da90c is 175, key is test_row_0/A:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:28,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569248441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569248442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569248444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742276_1452 (size=48389) 2024-11-14T07:26:28,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569248453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569248457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569248565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569248565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569248565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569248586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569248586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569248774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569248775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569248785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569248805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569248805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:28,864 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/28eba26a491e435199baac86323da90c 2024-11-14T07:26:28,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4922e3df7c6a4ddaacd2539f58996419 is 50, key is test_row_0/B:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:28,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:28,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742277_1453 (size=12151) 2024-11-14T07:26:28,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4922e3df7c6a4ddaacd2539f58996419 2024-11-14T07:26:28,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/e673d41f28ef4fde9eefe44e3111bec4 is 50, key is test_row_0/C:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:28,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742278_1454 (size=12151) 2024-11-14T07:26:28,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/e673d41f28ef4fde9eefe44e3111bec4 2024-11-14T07:26:28,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/28eba26a491e435199baac86323da90c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c 2024-11-14T07:26:28,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c, entries=250, sequenceid=226, filesize=47.3 K 2024-11-14T07:26:28,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4922e3df7c6a4ddaacd2539f58996419 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419 2024-11-14T07:26:28,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T07:26:28,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/e673d41f28ef4fde9eefe44e3111bec4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4 2024-11-14T07:26:28,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T07:26:28,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a7fb55e65d21ca143b07686010a18f00 in 626ms, sequenceid=226, compaction requested=true 2024-11-14T07:26:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:28,980 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:28,980 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:28,982 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:28,982 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:28,982 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:28,982 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:28,982 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,982 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,982 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/93ef9be7c1044ffd9df9bef7c891481d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=36.0 K 2024-11-14T07:26:28,982 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=108.4 K 2024-11-14T07:26:28,982 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:28,982 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c] 2024-11-14T07:26:28,983 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ef9be7c1044ffd9df9bef7c891481d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186785 2024-11-14T07:26:28,983 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1a0aa486d544a4b6a64976200fa83a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186785 2024-11-14T07:26:28,983 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0efba5527fad4e878efbe5bbca1eaf13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569187000 2024-11-14T07:26:28,983 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7986d0e9bdee4a3aa9fb431e4a244ac0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569187000 2024-11-14T07:26:28,983 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4922e3df7c6a4ddaacd2539f58996419, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187173 2024-11-14T07:26:28,984 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28eba26a491e435199baac86323da90c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187149 2024-11-14T07:26:28,994 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:28,995 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:28,995 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/37203f4995bc487184af9fa02f173497 is 50, key is test_row_0/B:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:29,013 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147d2da25c4c7f455199a38ddf949bd93b_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:29,015 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147d2da25c4c7f455199a38ddf949bd93b_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:29,015 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147d2da25c4c7f455199a38ddf949bd93b_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742279_1455 (size=12697) 2024-11-14T07:26:29,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742280_1456 (size=4469) 2024-11-14T07:26:29,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-14T07:26:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:29,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:26:29,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111466547d7a23e4436e8eab685c37fa8d32_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569188442/Put/seqid=0 2024-11-14T07:26:29,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742281_1457 (size=12304) 2024-11-14T07:26:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:29,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:29,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569249118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569249125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569249125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569249130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569249131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569249234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569249255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569249256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:29,436 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#387 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:29,437 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/f22a2889f897452eae77fde16103d0e3 is 175, key is test_row_0/A:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:29,438 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/37203f4995bc487184af9fa02f173497 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/37203f4995bc487184af9fa02f173497 2024-11-14T07:26:29,443 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into 37203f4995bc487184af9fa02f173497(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:29,443 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:29,443 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=13, startTime=1731569188980; duration=0sec 2024-11-14T07:26:29,443 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:29,443 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:29,443 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:29,444 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:29,444 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:29,444 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:29,444 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/02ad770ce1294e5aa1e739677f3d7c30, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=36.0 K 2024-11-14T07:26:29,445 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ad770ce1294e5aa1e739677f3d7c30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731569186785 2024-11-14T07:26:29,445 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d23a498da154b9a8706d994fa3c0977, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569187000 2024-11-14T07:26:29,445 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e673d41f28ef4fde9eefe44e3111bec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187173 2024-11-14T07:26:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569249445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742282_1458 (size=31651) 2024-11-14T07:26:29,460 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#389 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:29,461 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/e63a7b4aab8e43428858379019568f59 is 50, key is test_row_0/C:col10/1731569188353/Put/seqid=0 2024-11-14T07:26:29,463 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/f22a2889f897452eae77fde16103d0e3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3 2024-11-14T07:26:29,474 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into f22a2889f897452eae77fde16103d0e3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:29,474 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:29,474 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=13, startTime=1731569188980; duration=0sec 2024-11-14T07:26:29,474 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:29,474 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569249474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569249474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742283_1459 (size=12697) 2024-11-14T07:26:29,498 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111466547d7a23e4436e8eab685c37fa8d32_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111466547d7a23e4436e8eab685c37fa8d32_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:29,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/947027813dc342e9ac3996a2ffb7add0, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:29,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/947027813dc342e9ac3996a2ffb7add0 is 175, key is test_row_0/A:col10/1731569188442/Put/seqid=0 2024-11-14T07:26:29,506 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/e63a7b4aab8e43428858379019568f59 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e63a7b4aab8e43428858379019568f59 2024-11-14T07:26:29,512 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into e63a7b4aab8e43428858379019568f59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:29,512 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:29,512 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=13, startTime=1731569188980; duration=0sec 2024-11-14T07:26:29,512 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:29,512 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:29,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742284_1460 (size=31105) 2024-11-14T07:26:29,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/947027813dc342e9ac3996a2ffb7add0 2024-11-14T07:26:29,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4610c55255c6467dbf8b9577229f0eb3 is 50, key is test_row_0/B:col10/1731569188442/Put/seqid=0 2024-11-14T07:26:29,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742285_1461 (size=12151) 2024-11-14T07:26:29,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569249644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569249645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569249765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569249795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:29,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569249796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:30,007 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4610c55255c6467dbf8b9577229f0eb3 2024-11-14T07:26:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/eb0f4200191047fd801bc275dbd8c923 is 50, key is test_row_0/C:col10/1731569188442/Put/seqid=0 2024-11-14T07:26:30,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742286_1462 (size=12151) 2024-11-14T07:26:30,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569250284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569250304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:30,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569250314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:30,446 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/eb0f4200191047fd801bc275dbd8c923 2024-11-14T07:26:30,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/947027813dc342e9ac3996a2ffb7add0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0 2024-11-14T07:26:30,457 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0, entries=150, sequenceid=252, filesize=30.4 K 2024-11-14T07:26:30,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/4610c55255c6467dbf8b9577229f0eb3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3 2024-11-14T07:26:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T07:26:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/eb0f4200191047fd801bc275dbd8c923 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923 2024-11-14T07:26:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T07:26:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,491 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a7fb55e65d21ca143b07686010a18f00 in 1435ms, sequenceid=252, compaction requested=false 2024-11-14T07:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-14T07:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-14T07:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T07:26:30,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2890 sec 2024-11-14T07:26:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.2960 sec 2024-11-14T07:26:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:30,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142a04fde129704ba4945adc3adf036655_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742287_1463 (size=12454) 2024-11-14T07:26:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569250943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:30,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569250954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569251064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569251075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:31,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142a04fde129704ba4945adc3adf036655_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142a04fde129704ba4945adc3adf036655_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:31,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/94d45457b94843f3b8ce5fb050f7a2a9, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/94d45457b94843f3b8ce5fb050f7a2a9 is 175, key is test_row_0/A:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742288_1464 (size=31251) 2024-11-14T07:26:31,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/94d45457b94843f3b8ce5fb050f7a2a9 2024-11-14T07:26:31,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/88b19e70ac51489aad608c8225d7c768 is 50, key is test_row_0/B:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569251275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569251284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742289_1465 (size=9857) 2024-11-14T07:26:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569251306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569251314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/88b19e70ac51489aad608c8225d7c768 2024-11-14T07:26:31,321 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,321 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/04bdc2ae1859466a922d9e71ad57871b is 50, key is test_row_0/C:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569251334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,339 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742290_1466 (size=9857) 2024-11-14T07:26:31,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/04bdc2ae1859466a922d9e71ad57871b 2024-11-14T07:26:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/94d45457b94843f3b8ce5fb050f7a2a9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9 2024-11-14T07:26:31,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9, entries=150, sequenceid=266, filesize=30.5 K 2024-11-14T07:26:31,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/88b19e70ac51489aad608c8225d7c768 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768 2024-11-14T07:26:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T07:26:31,365 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:31,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-14T07:26:31,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:31,368 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:31,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:31,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768, entries=100, sequenceid=266, filesize=9.6 K 2024-11-14T07:26:31,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/04bdc2ae1859466a922d9e71ad57871b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b 2024-11-14T07:26:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T07:26:31,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b, entries=100, sequenceid=266, filesize=9.6 K 2024-11-14T07:26:31,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a7fb55e65d21ca143b07686010a18f00 in 676ms, sequenceid=266, compaction requested=true 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:31,378 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:31,378 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:31,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:31,384 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:31,384 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:31,384 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:31,384 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/37203f4995bc487184af9fa02f173497, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=33.9 K 2024-11-14T07:26:31,384 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:31,385 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:31,385 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:31,385 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=91.8 K 2024-11-14T07:26:31,386 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:31,386 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9] 2024-11-14T07:26:31,386 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 37203f4995bc487184af9fa02f173497, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187173 2024-11-14T07:26:31,386 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22a2889f897452eae77fde16103d0e3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187173 2024-11-14T07:26:31,386 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4610c55255c6467dbf8b9577229f0eb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731569188442 2024-11-14T07:26:31,387 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947027813dc342e9ac3996a2ffb7add0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731569188442 2024-11-14T07:26:31,387 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b19e70ac51489aad608c8225d7c768, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569189130 2024-11-14T07:26:31,387 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d45457b94843f3b8ce5fb050f7a2a9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569189117 2024-11-14T07:26:31,418 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:31,419 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/9557ea99da4c4e898c34ab94a1451a66 is 50, key is test_row_0/B:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,429 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,434 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b3f5449eecee4ddfbf0703b31d0d5c2f_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,437 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b3f5449eecee4ddfbf0703b31d0d5c2f_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,437 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3f5449eecee4ddfbf0703b31d0d5c2f_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742291_1467 (size=12899) 2024-11-14T07:26:31,464 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/9557ea99da4c4e898c34ab94a1451a66 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9557ea99da4c4e898c34ab94a1451a66 2024-11-14T07:26:31,470 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into 9557ea99da4c4e898c34ab94a1451a66(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:31,470 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:31,470 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=13, startTime=1731569191378; duration=0sec 2024-11-14T07:26:31,470 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:31,470 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:31,471 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:31,471 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:31,472 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:31,472 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:31,472 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e63a7b4aab8e43428858379019568f59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=33.9 K 2024-11-14T07:26:31,472 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e63a7b4aab8e43428858379019568f59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731569187173 2024-11-14T07:26:31,473 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0f4200191047fd801bc275dbd8c923, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731569188442 2024-11-14T07:26:31,475 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 04bdc2ae1859466a922d9e71ad57871b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569189130 2024-11-14T07:26:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T07:26:31,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742292_1468 (size=4469) 2024-11-14T07:26:31,497 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#396 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:31,498 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9c4f47f3fed04fa99b96f0897b5d76fe is 175, key is test_row_0/A:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,512 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#397 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:31,512 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/ba172d745bb4475ea254359756768373 is 50, key is test_row_0/C:col10/1731569190700/Put/seqid=0 2024-11-14T07:26:31,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742293_1469 (size=31960) 2024-11-14T07:26:31,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T07:26:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:31,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:31,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742294_1470 (size=12899) 2024-11-14T07:26:31,566 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/ba172d745bb4475ea254359756768373 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/ba172d745bb4475ea254359756768373 2024-11-14T07:26:31,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa8e1187b6dc495299c6f4b1345148d5_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569190952/Put/seqid=0 2024-11-14T07:26:31,571 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into ba172d745bb4475ea254359756768373(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:31,571 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:31,571 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=13, startTime=1731569191378; duration=0sec 2024-11-14T07:26:31,572 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:31,572 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:31,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742295_1471 (size=12454) 2024-11-14T07:26:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:31,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:31,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569251628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569251633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T07:26:31,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569251744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569251744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,918 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/9c4f47f3fed04fa99b96f0897b5d76fe as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe 2024-11-14T07:26:31,922 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into 9c4f47f3fed04fa99b96f0897b5d76fe(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:31,922 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:31,922 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=13, startTime=1731569191378; duration=0sec 2024-11-14T07:26:31,922 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:31,922 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569251954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569251954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:31,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:31,995 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa8e1187b6dc495299c6f4b1345148d5_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa8e1187b6dc495299c6f4b1345148d5_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:31,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7291231e61a44ed886a1dcc5603d5bfd, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7291231e61a44ed886a1dcc5603d5bfd is 175, key is test_row_0/A:col10/1731569190952/Put/seqid=0 2024-11-14T07:26:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T07:26:32,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742296_1472 (size=31255) 2024-11-14T07:26:32,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7291231e61a44ed886a1dcc5603d5bfd 2024-11-14T07:26:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7830a142ab284f50b1daec529fad95c7 is 50, key is test_row_0/B:col10/1731569190952/Put/seqid=0 2024-11-14T07:26:32,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742297_1473 (size=12301) 2024-11-14T07:26:32,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569252264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569252265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:32,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7830a142ab284f50b1daec529fad95c7 2024-11-14T07:26:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/4254fea032c04f71bc12b7f639d38379 is 50, key is test_row_0/C:col10/1731569190952/Put/seqid=0 2024-11-14T07:26:32,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742298_1474 (size=12301) 2024-11-14T07:26:32,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/4254fea032c04f71bc12b7f639d38379 2024-11-14T07:26:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/7291231e61a44ed886a1dcc5603d5bfd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd 2024-11-14T07:26:32,464 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd, entries=150, sequenceid=292, filesize=30.5 K 2024-11-14T07:26:32,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/7830a142ab284f50b1daec529fad95c7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7 2024-11-14T07:26:32,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T07:26:32,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/4254fea032c04f71bc12b7f639d38379 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379 2024-11-14T07:26:32,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T07:26:32,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a7fb55e65d21ca143b07686010a18f00 in 951ms, sequenceid=292, compaction requested=false 2024-11-14T07:26:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-14T07:26:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-14T07:26:32,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T07:26:32,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1130 sec 2024-11-14T07:26:32,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1180 sec 2024-11-14T07:26:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T07:26:32,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:32,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T07:26:32,519 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:32,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:32,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:32,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T07:26:32,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:32,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f4e1eec435b646fbbc6f6c2bf565baca_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569191621/Put/seqid=0 2024-11-14T07:26:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742299_1475 (size=12454) 2024-11-14T07:26:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:32,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:32,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569252905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:32,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569252905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:32,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569253026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569253026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:33,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f4e1eec435b646fbbc6f6c2bf565baca_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4e1eec435b646fbbc6f6c2bf565baca_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/eebdb4ae105a4bbea82fe92ea95b126e, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/eebdb4ae105a4bbea82fe92ea95b126e is 175, key is test_row_0/A:col10/1731569191621/Put/seqid=0 2024-11-14T07:26:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742300_1476 (size=31255) 2024-11-14T07:26:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569253234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569253234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33784 deadline: 1731569253326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,332 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33858 deadline: 1731569253345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33828 deadline: 1731569253347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,356 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:33,356 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in a7fb55e65d21ca143b07686010a18f00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,555 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/eebdb4ae105a4bbea82fe92ea95b126e 2024-11-14T07:26:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569253554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569253555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:33,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2691d524e021496eaf39f65e9e289e3c is 50, key is test_row_0/B:col10/1731569191621/Put/seqid=0 2024-11-14T07:26:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742301_1477 (size=12301) 2024-11-14T07:26:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:34,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2691d524e021496eaf39f65e9e289e3c 2024-11-14T07:26:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2ddded26a099474ca3dd5ab0a4cfeb49 is 50, key is test_row_0/C:col10/1731569191621/Put/seqid=0 2024-11-14T07:26:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742302_1478 (size=12301) 2024-11-14T07:26:34,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2ddded26a099474ca3dd5ab0a4cfeb49 2024-11-14T07:26:34,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/eebdb4ae105a4bbea82fe92ea95b126e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e 2024-11-14T07:26:34,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e, entries=150, sequenceid=305, filesize=30.5 K 2024-11-14T07:26:34,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2691d524e021496eaf39f65e9e289e3c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c 2024-11-14T07:26:34,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-14T07:26:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/2ddded26a099474ca3dd5ab0a4cfeb49 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49 2024-11-14T07:26:34,062 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49, entries=150, sequenceid=305, filesize=12.0 K 2024-11-14T07:26:34,063 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a7fb55e65d21ca143b07686010a18f00 in 1388ms, sequenceid=305, compaction requested=true 2024-11-14T07:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T07:26:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T07:26:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:34,072 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:34,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:26:34,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T07:26:34,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-14T07:26:34,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5590 sec 2024-11-14T07:26:34,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eda192bb85934bad8dc5855c42b0e2a4_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569254119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742303_1479 (size=14994) 2024-11-14T07:26:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569254125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569254245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569254245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569254455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569254455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:34,539 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eda192bb85934bad8dc5855c42b0e2a4_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eda192bb85934bad8dc5855c42b0e2a4_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:34,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/2171e072d052410f92cdcb4fa23e5f63, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:34,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/2171e072d052410f92cdcb4fa23e5f63 is 175, key is test_row_0/A:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:34,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742304_1480 (size=39949) 2024-11-14T07:26:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T07:26:34,664 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:34,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T07:26:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:34,666 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:34,667 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:34,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569254774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569254775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:34,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T07:26:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:34,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/2171e072d052410f92cdcb4fa23e5f63 2024-11-14T07:26:34,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/1d92dcb9e7bc473898ab744554ffd2b3 is 50, key is test_row_0/B:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:34,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742305_1481 (size=12301) 2024-11-14T07:26:34,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T07:26:34,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:34,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:34,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:34,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:35,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T07:26:35,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:35,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T07:26:35,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33844 deadline: 1731569255295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1731569255295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:26:35,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/1d92dcb9e7bc473898ab744554ffd2b3 2024-11-14T07:26:35,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/db9e6b814c824415a5d1740f0db4e3ac is 50, key is test_row_0/C:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:35,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742306_1482 (size=12301) 2024-11-14T07:26:35,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/db9e6b814c824415a5d1740f0db4e3ac 2024-11-14T07:26:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/2171e072d052410f92cdcb4fa23e5f63 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63 2024-11-14T07:26:35,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63, entries=200, sequenceid=331, filesize=39.0 K 2024-11-14T07:26:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/1d92dcb9e7bc473898ab744554ffd2b3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3 2024-11-14T07:26:35,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T07:26:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/db9e6b814c824415a5d1740f0db4e3ac as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac 2024-11-14T07:26:35,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T07:26:35,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a7fb55e65d21ca143b07686010a18f00 in 1326ms, sequenceid=331, compaction requested=true 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:35,398 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:35,398 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7fb55e65d21ca143b07686010a18f00:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:35,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:35,399 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:35,399 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134419 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:35,399 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/B is initiating minor compaction (all files) 2024-11-14T07:26:35,399 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/A is initiating minor compaction (all files) 2024-11-14T07:26:35,399 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/B in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,399 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/A in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,399 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=131.3 K 2024-11-14T07:26:35,399 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9557ea99da4c4e898c34ab94a1451a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=48.6 K 2024-11-14T07:26:35,399 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,399 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63] 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9557ea99da4c4e898c34ab94a1451a66, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569188449 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4f47f3fed04fa99b96f0897b5d76fe, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569188449 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7830a142ab284f50b1daec529fad95c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731569190938 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7291231e61a44ed886a1dcc5603d5bfd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731569190938 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2691d524e021496eaf39f65e9e289e3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731569191621 2024-11-14T07:26:35,400 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting eebdb4ae105a4bbea82fe92ea95b126e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731569191621 2024-11-14T07:26:35,401 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d92dcb9e7bc473898ab744554ffd2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731569192901 2024-11-14T07:26:35,401 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2171e072d052410f92cdcb4fa23e5f63, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731569192887 2024-11-14T07:26:35,421 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:35,430 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:35,431 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/a791f9bd737a46b2b7c2ddffdbab407f is 50, key is test_row_0/B:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:35,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T07:26:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T07:26:35,445 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111416e240a93f7a4946a00db2bc21037f4f_a7fb55e65d21ca143b07686010a18f00 store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:35,448 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111416e240a93f7a4946a00db2bc21037f4f_a7fb55e65d21ca143b07686010a18f00, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:35,448 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111416e240a93f7a4946a00db2bc21037f4f_a7fb55e65d21ca143b07686010a18f00 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:35,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742307_1483 (size=13085) 2024-11-14T07:26:35,463 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/a791f9bd737a46b2b7c2ddffdbab407f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/a791f9bd737a46b2b7c2ddffdbab407f 2024-11-14T07:26:35,479 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/B of a7fb55e65d21ca143b07686010a18f00 into a791f9bd737a46b2b7c2ddffdbab407f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:35,480 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:35,480 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/B, priority=12, startTime=1731569195398; duration=0sec 2024-11-14T07:26:35,480 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:35,480 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:B 2024-11-14T07:26:35,480 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:35,481 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:35,482 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): a7fb55e65d21ca143b07686010a18f00/C is initiating minor compaction (all files) 2024-11-14T07:26:35,482 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7fb55e65d21ca143b07686010a18f00/C in TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:35,482 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/ba172d745bb4475ea254359756768373, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp, totalSize=48.6 K 2024-11-14T07:26:35,483 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ba172d745bb4475ea254359756768373, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731569188449 2024-11-14T07:26:35,484 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4254fea032c04f71bc12b7f639d38379, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731569190938 2024-11-14T07:26:35,484 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ddded26a099474ca3dd5ab0a4cfeb49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731569191621 2024-11-14T07:26:35,484 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting db9e6b814c824415a5d1740f0db4e3ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731569192901 2024-11-14T07:26:35,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144cf583631f09422a9f5c84b3d830c2cc_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569194081/Put/seqid=0 2024-11-14T07:26:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742308_1484 (size=4469) 2024-11-14T07:26:35,496 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#A#compaction#407 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:35,497 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a1c4c408e992416d96f3316a9e6c368b is 175, key is test_row_0/A:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:35,502 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7fb55e65d21ca143b07686010a18f00#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:35,502 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/5d23c7d735954be9854448f0ae8eee10 is 50, key is test_row_0/C:col10/1731569194068/Put/seqid=0 2024-11-14T07:26:35,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742309_1485 (size=32039) 2024-11-14T07:26:35,536 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a1c4c408e992416d96f3316a9e6c368b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a1c4c408e992416d96f3316a9e6c368b 2024-11-14T07:26:35,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742310_1486 (size=12454) 2024-11-14T07:26:35,545 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/A of a7fb55e65d21ca143b07686010a18f00 into a1c4c408e992416d96f3316a9e6c368b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:35,546 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:35,546 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/A, priority=12, startTime=1731569195398; duration=0sec 2024-11-14T07:26:35,546 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:35,546 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:A 2024-11-14T07:26:35,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742311_1487 (size=13085) 2024-11-14T07:26:35,568 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/5d23c7d735954be9854448f0ae8eee10 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/5d23c7d735954be9854448f0ae8eee10 2024-11-14T07:26:35,576 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7fb55e65d21ca143b07686010a18f00/C of a7fb55e65d21ca143b07686010a18f00 into 5d23c7d735954be9854448f0ae8eee10(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:35,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:35,576 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., storeName=a7fb55e65d21ca143b07686010a18f00/C, priority=12, startTime=1731569195398; duration=0sec 2024-11-14T07:26:35,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:35,577 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7fb55e65d21ca143b07686010a18f00:C 2024-11-14T07:26:35,582 INFO [Thread-1765 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1765. 2024-11-14T07:26:35,582 DEBUG [Thread-1765 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:35,582 DEBUG [Thread-1765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,583 DEBUG [Thread-1765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:35,585 INFO [Thread-1759 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1759. 2024-11-14T07:26:35,585 DEBUG [Thread-1759 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:35,585 DEBUG [Thread-1759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,585 INFO [Thread-1767 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1767. 2024-11-14T07:26:35,585 DEBUG [Thread-1759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:35,586 INFO [Thread-1763 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1763. 2024-11-14T07:26:35,586 DEBUG [Thread-1763 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:35,586 DEBUG [Thread-1763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,587 DEBUG [Thread-1763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:35,587 DEBUG [Thread-1767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:35,587 DEBUG [Thread-1767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,587 DEBUG [Thread-1767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,589 INFO [Thread-1761 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1761. 2024-11-14T07:26:35,589 DEBUG [Thread-1761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:35,589 DEBUG [Thread-1761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,589 DEBUG [Thread-1761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:35,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:35,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:35,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:35,944 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144cf583631f09422a9f5c84b3d830c2cc_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf583631f09422a9f5c84b3d830c2cc_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:35,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a9014660e7fe4710ab196958f26ccdcf, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a9014660e7fe4710ab196958f26ccdcf is 175, key is test_row_0/A:col10/1731569194081/Put/seqid=0 2024-11-14T07:26:35,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742312_1488 (size=31255) 2024-11-14T07:26:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:36,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. as already flushing 2024-11-14T07:26:36,317 INFO [Thread-1754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1754. 2024-11-14T07:26:36,317 DEBUG [Thread-1754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:36,317 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:36,317 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:36,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:36,326 INFO [Thread-1756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1756. 2024-11-14T07:26:36,326 DEBUG [Thread-1756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:36,327 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:36,327 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:36,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:36,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a9014660e7fe4710ab196958f26ccdcf 2024-11-14T07:26:36,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2569d6c448854a91a610c8d95ff46dd8 is 50, key is test_row_0/B:col10/1731569194081/Put/seqid=0 2024-11-14T07:26:36,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742313_1489 (size=12301) 2024-11-14T07:26:36,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2569d6c448854a91a610c8d95ff46dd8 2024-11-14T07:26:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/78cc818183ee4723b32c129d95eb81db is 50, key is test_row_0/C:col10/1731569194081/Put/seqid=0 2024-11-14T07:26:36,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742314_1490 (size=12301) 2024-11-14T07:26:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:37,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/78cc818183ee4723b32c129d95eb81db 2024-11-14T07:26:37,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/a9014660e7fe4710ab196958f26ccdcf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a9014660e7fe4710ab196958f26ccdcf 2024-11-14T07:26:37,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a9014660e7fe4710ab196958f26ccdcf, entries=150, sequenceid=341, filesize=30.5 K 2024-11-14T07:26:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/2569d6c448854a91a610c8d95ff46dd8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2569d6c448854a91a610c8d95ff46dd8 2024-11-14T07:26:37,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2569d6c448854a91a610c8d95ff46dd8, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T07:26:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/78cc818183ee4723b32c129d95eb81db as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/78cc818183ee4723b32c129d95eb81db 2024-11-14T07:26:37,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/78cc818183ee4723b32c129d95eb81db, entries=150, sequenceid=341, filesize=12.0 K 2024-11-14T07:26:37,201 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=13.42 KB/13740 for a7fb55e65d21ca143b07686010a18f00 in 1767ms, sequenceid=341, compaction requested=false 2024-11-14T07:26:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for a7fb55e65d21ca143b07686010a18f00: 2024-11-14T07:26:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T07:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T07:26:37,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T07:26:37,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-14T07:26:37,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5390 sec 2024-11-14T07:26:37,357 INFO [Thread-1748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1748. 2024-11-14T07:26:37,357 DEBUG [Thread-1748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:37,357 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,357 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:37,366 INFO [Thread-1750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1750. 2024-11-14T07:26:37,366 DEBUG [Thread-1750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:37,367 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,367 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:37,398 INFO [Thread-1752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1752. 2024-11-14T07:26:37,398 DEBUG [Thread-1752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:26:37,398 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,398 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:37,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T07:26:38,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-14T07:26:38,836 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1284 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3849 rows 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3867 rows 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1303 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3903 rows 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1287 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3858 rows 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1295 2024-11-14T07:26:38,837 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3884 rows 2024-11-14T07:26:38,837 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:26:38,838 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:26:38,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:38,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:38,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:26:38,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:26:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:38,847 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569198847"}]},"ts":"1731569198847"} 2024-11-14T07:26:38,849 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:26:38,849 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:26:38,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:26:38,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, UNASSIGN}] 2024-11-14T07:26:38,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, UNASSIGN 2024-11-14T07:26:38,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:38,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, UNASSIGN because future has completed 2024-11-14T07:26:38,856 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:26:38,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:39,010 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:39,010 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:26:39,010 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing a7fb55e65d21ca143b07686010a18f00, disabling compactions & flushes 2024-11-14T07:26:39,010 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:39,010 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:39,011 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. after waiting 0 ms 2024-11-14T07:26:39,011 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:39,011 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing a7fb55e65d21ca143b07686010a18f00 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T07:26:39,021 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111499db69054be347ff9245d2350b4269a8_a7fb55e65d21ca143b07686010a18f00 is 50, key is test_row_0/A:col10/1731569197355/Put/seqid=0 2024-11-14T07:26:39,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742315_1491 (size=12454) 2024-11-14T07:26:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:39,426 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:39,435 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111499db69054be347ff9245d2350b4269a8_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111499db69054be347ff9245d2350b4269a8_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:39,436 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/ab08adadbb534b35818d20c4e427138e, store: [table=TestAcidGuarantees family=A region=a7fb55e65d21ca143b07686010a18f00] 2024-11-14T07:26:39,436 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/ab08adadbb534b35818d20c4e427138e is 175, key is test_row_0/A:col10/1731569197355/Put/seqid=0 2024-11-14T07:26:39,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742316_1492 (size=31255) 2024-11-14T07:26:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:39,840 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/ab08adadbb534b35818d20c4e427138e 2024-11-14T07:26:39,849 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/f1c16890c1314dad9e4a4f40e50a358b is 50, key is test_row_0/B:col10/1731569197355/Put/seqid=0 2024-11-14T07:26:39,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742317_1493 (size=12301) 2024-11-14T07:26:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:40,254 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/f1c16890c1314dad9e4a4f40e50a358b 2024-11-14T07:26:40,267 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/3baafce1e0654cadacde29cfc17490d3 is 50, key is test_row_0/C:col10/1731569197355/Put/seqid=0 2024-11-14T07:26:40,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742318_1494 (size=12301) 2024-11-14T07:26:40,672 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/3baafce1e0654cadacde29cfc17490d3 2024-11-14T07:26:40,680 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/A/ab08adadbb534b35818d20c4e427138e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/ab08adadbb534b35818d20c4e427138e 2024-11-14T07:26:40,685 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/ab08adadbb534b35818d20c4e427138e, entries=150, sequenceid=352, filesize=30.5 K 2024-11-14T07:26:40,686 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/B/f1c16890c1314dad9e4a4f40e50a358b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/f1c16890c1314dad9e4a4f40e50a358b 2024-11-14T07:26:40,690 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/f1c16890c1314dad9e4a4f40e50a358b, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T07:26:40,690 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/.tmp/C/3baafce1e0654cadacde29cfc17490d3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/3baafce1e0654cadacde29cfc17490d3 2024-11-14T07:26:40,694 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/3baafce1e0654cadacde29cfc17490d3, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T07:26:40,695 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a7fb55e65d21ca143b07686010a18f00 in 1684ms, sequenceid=352, compaction requested=true 2024-11-14T07:26:40,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63] to archive 2024-11-14T07:26:40,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:40,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/5bed58c6ec6745d2bd45b8ca08e9cf1f 2024-11-14T07:26:40,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/80706b294c114a43a3db7ab5ad678523 2024-11-14T07:26:40,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3b2739e526d5491a801152411640e0e7 2024-11-14T07:26:40,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/de86356b27024d61af95a1a33dd21c9b 2024-11-14T07:26:40,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3df1960596f642b4b5421e4e55fd536e 2024-11-14T07:26:40,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/4b1443ff4bbf4bd58651e4cf9469f3c8 2024-11-14T07:26:40,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/8beb009124be455a914d21e631967e19 2024-11-14T07:26:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/46a003e63bb3458aa02c33fa967f9459 2024-11-14T07:26:40,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9f2f877c2f2d4ffb929d2741e464aaa7 2024-11-14T07:26:40,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/75eea87d6bec4a6281a74801e233cb85 2024-11-14T07:26:40,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/3f306226ecc842d586e54c30fd14b1ba 2024-11-14T07:26:40,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/79fd7ac48941456aae7f9159b42db8fa 2024-11-14T07:26:40,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/c6b583ad22c34f00a7df690c011de3f3 2024-11-14T07:26:40,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/0f1a0aa486d544a4b6a64976200fa83a 2024-11-14T07:26:40,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7986d0e9bdee4a3aa9fb431e4a244ac0 2024-11-14T07:26:40,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/28eba26a491e435199baac86323da90c 2024-11-14T07:26:40,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/f22a2889f897452eae77fde16103d0e3 2024-11-14T07:26:40,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/947027813dc342e9ac3996a2ffb7add0 2024-11-14T07:26:40,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/9c4f47f3fed04fa99b96f0897b5d76fe 2024-11-14T07:26:40,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/94d45457b94843f3b8ce5fb050f7a2a9 2024-11-14T07:26:40,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/7291231e61a44ed886a1dcc5603d5bfd 2024-11-14T07:26:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/eebdb4ae105a4bbea82fe92ea95b126e 2024-11-14T07:26:40,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/2171e072d052410f92cdcb4fa23e5f63 2024-11-14T07:26:40,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9456a6c2f15241ddb34c7d8564021bc7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fd267f2748fe42b9bb1e71d1c8fffd32, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/465eb78fde8e4bf88205bac0e05c22ff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/93ef9be7c1044ffd9df9bef7c891481d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/37203f4995bc487184af9fa02f173497, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9557ea99da4c4e898c34ab94a1451a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3] to archive 2024-11-14T07:26:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/dfdbac8af5b24b46853099c88b3d28a5 2024-11-14T07:26:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b830743519044f649c11a2a73a9c5c7c 2024-11-14T07:26:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9456a6c2f15241ddb34c7d8564021bc7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9456a6c2f15241ddb34c7d8564021bc7 2024-11-14T07:26:40,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/79cb14920f424acfa6decadea38e22a7 2024-11-14T07:26:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/13bd2208ee7e425aa12570747cfe8b1d 2024-11-14T07:26:40,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fd267f2748fe42b9bb1e71d1c8fffd32 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fd267f2748fe42b9bb1e71d1c8fffd32 2024-11-14T07:26:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/b67ddf720a52421fadd78e372dc0fc93 2024-11-14T07:26:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/bca7a0550ca349aab49d10b334dbf41e 2024-11-14T07:26:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/465eb78fde8e4bf88205bac0e05c22ff to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/465eb78fde8e4bf88205bac0e05c22ff 2024-11-14T07:26:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7991501379a84c4e99b9f4d3c79f8dd8 2024-11-14T07:26:40,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/80f1530771bd403f9aabcc056c3416e6 2024-11-14T07:26:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/fe6b2b1ca8b04ea487fe4416d5062038 2024-11-14T07:26:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/158ffb53db1a4fb0a22b81b9f3602208 2024-11-14T07:26:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/93ef9be7c1044ffd9df9bef7c891481d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/93ef9be7c1044ffd9df9bef7c891481d 2024-11-14T07:26:40,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/0efba5527fad4e878efbe5bbca1eaf13 2024-11-14T07:26:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/37203f4995bc487184af9fa02f173497 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/37203f4995bc487184af9fa02f173497 2024-11-14T07:26:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4922e3df7c6a4ddaacd2539f58996419 2024-11-14T07:26:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/4610c55255c6467dbf8b9577229f0eb3 2024-11-14T07:26:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9557ea99da4c4e898c34ab94a1451a66 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/9557ea99da4c4e898c34ab94a1451a66 2024-11-14T07:26:40,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/88b19e70ac51489aad608c8225d7c768 2024-11-14T07:26:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/7830a142ab284f50b1daec529fad95c7 2024-11-14T07:26:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2691d524e021496eaf39f65e9e289e3c 2024-11-14T07:26:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/1d92dcb9e7bc473898ab744554ffd2b3 2024-11-14T07:26:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/102684a4a87748d7972691e375604480, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/c53ba964cbad4500b3d745d53d89f669, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1d4617bf40b948458ac1475456ef0a88, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/02ad770ce1294e5aa1e739677f3d7c30, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e63a7b4aab8e43428858379019568f59, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/ba172d745bb4475ea254359756768373, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac] to archive 2024-11-14T07:26:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:40,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/8158afa0316048c687a52cc8553ea974 2024-11-14T07:26:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/19e97061f6764141b1b25277d1193dad 2024-11-14T07:26:40,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/102684a4a87748d7972691e375604480 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/102684a4a87748d7972691e375604480 2024-11-14T07:26:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/897fc1f419494684b4e9f540788696aa 2024-11-14T07:26:40,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/79b8fdcb241f462c9959cebc09850157 2024-11-14T07:26:40,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/c53ba964cbad4500b3d745d53d89f669 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/c53ba964cbad4500b3d745d53d89f669 2024-11-14T07:26:40,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/d4ac486ae2624bcba66efaaa2982042f 2024-11-14T07:26:40,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1e917d0128cc41e696a32242bdd5c2c7 2024-11-14T07:26:40,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1d4617bf40b948458ac1475456ef0a88 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/1d4617bf40b948458ac1475456ef0a88 2024-11-14T07:26:40,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/dd45615773024ccd9eec2b1f1bcc8937 2024-11-14T07:26:40,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/0f5a4c4f47bd4d848a57519943f9ebcd 2024-11-14T07:26:40,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/9d13947f49174ef3ad9bc648aa2a1aa5 2024-11-14T07:26:40,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/02ad770ce1294e5aa1e739677f3d7c30 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/02ad770ce1294e5aa1e739677f3d7c30 2024-11-14T07:26:40,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/b7f1a8ff0f00420e89ec2bed25ce329c 2024-11-14T07:26:40,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2d23a498da154b9a8706d994fa3c0977 2024-11-14T07:26:40,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e63a7b4aab8e43428858379019568f59 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e63a7b4aab8e43428858379019568f59 2024-11-14T07:26:40,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/e673d41f28ef4fde9eefe44e3111bec4 2024-11-14T07:26:40,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/eb0f4200191047fd801bc275dbd8c923 2024-11-14T07:26:40,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/ba172d745bb4475ea254359756768373 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/ba172d745bb4475ea254359756768373 2024-11-14T07:26:40,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/04bdc2ae1859466a922d9e71ad57871b 2024-11-14T07:26:40,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/4254fea032c04f71bc12b7f639d38379 2024-11-14T07:26:40,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/2ddded26a099474ca3dd5ab0a4cfeb49 2024-11-14T07:26:40,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/db9e6b814c824415a5d1740f0db4e3ac 2024-11-14T07:26:40,782 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits/355.seqid, newMaxSeqId=355, maxSeqId=4 2024-11-14T07:26:40,782 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. 2024-11-14T07:26:40,782 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for a7fb55e65d21ca143b07686010a18f00: Waiting for close lock at 1731569199010Running coprocessor pre-close hooks at 1731569199010Disabling compacts and flushes for region at 1731569199010Disabling writes for close at 1731569199011 (+1 ms)Obtaining lock to block concurrent updates at 1731569199011Preparing flush snapshotting stores in a7fb55e65d21ca143b07686010a18f00 at 1731569199011Finished memstore snapshotting TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731569199012 (+1 ms)Flushing stores of TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00. at 1731569199013 (+1 ms)Flushing a7fb55e65d21ca143b07686010a18f00/A: creating writer at 1731569199013Flushing mob file a7fb55e65d21ca143b07686010a18f00/A: appending metadata at 1731569199021 (+8 ms)Flushing mob file a7fb55e65d21ca143b07686010a18f00/A: closing flushed file at 1731569199021Flushing a7fb55e65d21ca143b07686010a18f00/A: appending metadata at 1731569199436 (+415 ms)Flushing a7fb55e65d21ca143b07686010a18f00/A: closing flushed file at 1731569199436Flushing a7fb55e65d21ca143b07686010a18f00/B: creating writer at 1731569199845 (+409 ms)Flushing a7fb55e65d21ca143b07686010a18f00/B: appending metadata at 1731569199849 (+4 ms)Flushing a7fb55e65d21ca143b07686010a18f00/B: closing flushed file at 1731569199849Flushing a7fb55e65d21ca143b07686010a18f00/C: creating writer at 1731569200262 (+413 ms)Flushing a7fb55e65d21ca143b07686010a18f00/C: appending metadata at 1731569200266 (+4 ms)Flushing a7fb55e65d21ca143b07686010a18f00/C: closing flushed file at 1731569200266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93468b7: reopening flushed file at 1731569200679 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e30a2d1: reopening flushed file at 1731569200685 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52852a3c: reopening flushed file at 1731569200690 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a7fb55e65d21ca143b07686010a18f00 in 1684ms, sequenceid=352, compaction requested=true at 1731569200695 (+5 ms)Writing region close event to WAL at 1731569200780 (+85 ms)Running coprocessor post-close hooks at 1731569200782 (+2 ms)Closed at 1731569200782 2024-11-14T07:26:40,784 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:40,785 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=a7fb55e65d21ca143b07686010a18f00, regionState=CLOSED 2024-11-14T07:26:40,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:40,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T07:26:40,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure a7fb55e65d21ca143b07686010a18f00, server=c7aa6e2790b3,46235,1731569085523 in 1.9300 sec 2024-11-14T07:26:40,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-14T07:26:40,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7fb55e65d21ca143b07686010a18f00, UNASSIGN in 1.9350 sec 2024-11-14T07:26:40,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T07:26:40,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9390 sec 2024-11-14T07:26:40,791 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569200791"}]},"ts":"1731569200791"} 2024-11-14T07:26:40,792 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:26:40,792 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:26:40,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-14T07:26:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T07:26:40,996 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:40,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:26:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:40,999 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:26:41,000 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits] 2024-11-14T07:26:41,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a1c4c408e992416d96f3316a9e6c368b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a1c4c408e992416d96f3316a9e6c368b 2024-11-14T07:26:41,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a9014660e7fe4710ab196958f26ccdcf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/a9014660e7fe4710ab196958f26ccdcf 2024-11-14T07:26:41,008 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/ab08adadbb534b35818d20c4e427138e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/A/ab08adadbb534b35818d20c4e427138e 2024-11-14T07:26:41,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2569d6c448854a91a610c8d95ff46dd8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/2569d6c448854a91a610c8d95ff46dd8 2024-11-14T07:26:41,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/a791f9bd737a46b2b7c2ddffdbab407f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/a791f9bd737a46b2b7c2ddffdbab407f 2024-11-14T07:26:41,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/f1c16890c1314dad9e4a4f40e50a358b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/B/f1c16890c1314dad9e4a4f40e50a358b 2024-11-14T07:26:41,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/3baafce1e0654cadacde29cfc17490d3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/3baafce1e0654cadacde29cfc17490d3 2024-11-14T07:26:41,016 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/5d23c7d735954be9854448f0ae8eee10 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/5d23c7d735954be9854448f0ae8eee10 2024-11-14T07:26:41,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/78cc818183ee4723b32c129d95eb81db to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/C/78cc818183ee4723b32c129d95eb81db 2024-11-14T07:26:41,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits/355.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00/recovered.edits/355.seqid 2024-11-14T07:26:41,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,021 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:26:41,022 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:26:41,022 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:26:41,026 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401dc4d6ff25d4d8ab66544392d9b18e2_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111401dc4d6ff25d4d8ab66544392d9b18e2_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,027 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b37cfd87c9e4da7987e2ac353aa109f_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140b37cfd87c9e4da7987e2ac353aa109f_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,029 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f8a1842f56b4c0181e54e3b870c28cf_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f8a1842f56b4c0181e54e3b870c28cf_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,030 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111428e0549ed9df49fc9fbb391aa145c3ef_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111428e0549ed9df49fc9fbb391aa145c3ef_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,031 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142a04fde129704ba4945adc3adf036655_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142a04fde129704ba4945adc3adf036655_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,033 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf583631f09422a9f5c84b3d830c2cc_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144cf583631f09422a9f5c84b3d830c2cc_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,034 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462370c2f986841248f27a6681b1fb7de_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462370c2f986841248f27a6681b1fb7de_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,035 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111466547d7a23e4436e8eab685c37fa8d32_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111466547d7a23e4436e8eab685c37fa8d32_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,036 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111499db69054be347ff9245d2350b4269a8_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111499db69054be347ff9245d2350b4269a8_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,037 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b38ce2be73f0488aabea0c338a91b2b3_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b38ce2be73f0488aabea0c338a91b2b3_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,038 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dcb149745d49aba47886c881c3a03b_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3dcb149745d49aba47886c881c3a03b_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,039 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c6393921b74840f58b0d7b5050eadab4_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c6393921b74840f58b0d7b5050eadab4_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,039 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db8c4eed5d4941cd859a3a6ad53fbca8_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114db8c4eed5d4941cd859a3a6ad53fbca8_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,040 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eda192bb85934bad8dc5855c42b0e2a4_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eda192bb85934bad8dc5855c42b0e2a4_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,041 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f34575229c0a4eb48b75492c38361101_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f34575229c0a4eb48b75492c38361101_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,042 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4e1eec435b646fbbc6f6c2bf565baca_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f4e1eec435b646fbbc6f6c2bf565baca_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,043 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8dd851c414a48e9b4a1915c4103dae5_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8dd851c414a48e9b4a1915c4103dae5_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,044 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa8e1187b6dc495299c6f4b1345148d5_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa8e1187b6dc495299c6f4b1345148d5_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fabfd0aba36747719c2c39862a780cce_a7fb55e65d21ca143b07686010a18f00 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fabfd0aba36747719c2c39862a780cce_a7fb55e65d21ca143b07686010a18f00 2024-11-14T07:26:41,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:26:41,047 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,049 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:26:41,051 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:26:41,052 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,052 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:26:41,053 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569201052"}]},"ts":"9223372036854775807"} 2024-11-14T07:26:41,054 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:26:41,054 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a7fb55e65d21ca143b07686010a18f00, NAME => 'TestAcidGuarantees,,1731569172439.a7fb55e65d21ca143b07686010a18f00.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:26:41,054 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:26:41,054 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569201054"}]},"ts":"9223372036854775807"} 2024-11-14T07:26:41,056 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:26:41,056 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-14T07:26:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T07:26:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:26:41,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:41,117 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 242), OpenFileDescriptor=459 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 475) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5598 (was 5828) 2024-11-14T07:26:41,128 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=5598 2024-11-14T07:26:41,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:26:41,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:26:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:26:41,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:26:41,133 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:41,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-14T07:26:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T07:26:41,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:26:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742319_1495 (size=959) 2024-11-14T07:26:41,142 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:26:41,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742320_1496 (size=53) 2024-11-14T07:26:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T07:26:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5f1c5a6a1ccc8a28937c7c671893d5fc, disabling compactions & flushes 2024-11-14T07:26:41,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. after waiting 0 ms 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: Waiting for close lock at 1731569201548Disabling compacts and flushes for region at 1731569201548Disabling writes for close at 1731569201548Writing region close event to WAL at 1731569201548Closed at 1731569201548 2024-11-14T07:26:41,549 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:26:41,549 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569201549"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569201549"}]},"ts":"1731569201549"} 2024-11-14T07:26:41,551 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:26:41,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:26:41,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569201552"}]},"ts":"1731569201552"} 2024-11-14T07:26:41,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:26:41,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, ASSIGN}] 2024-11-14T07:26:41,555 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, ASSIGN 2024-11-14T07:26:41,556 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:26:41,707 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=5f1c5a6a1ccc8a28937c7c671893d5fc, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:41,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, ASSIGN because future has completed 2024-11-14T07:26:41,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:26:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T07:26:41,872 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,872 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:26:41,874 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,874 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:26:41,875 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,875 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,877 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,879 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1c5a6a1ccc8a28937c7c671893d5fc columnFamilyName A 2024-11-14T07:26:41,879 DEBUG [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:41,880 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(327): Store=5f1c5a6a1ccc8a28937c7c671893d5fc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:41,880 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,881 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1c5a6a1ccc8a28937c7c671893d5fc columnFamilyName B 2024-11-14T07:26:41,881 DEBUG [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:41,882 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(327): Store=5f1c5a6a1ccc8a28937c7c671893d5fc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:41,882 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,883 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f1c5a6a1ccc8a28937c7c671893d5fc columnFamilyName C 2024-11-14T07:26:41,883 DEBUG [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:26:41,883 INFO [StoreOpener-5f1c5a6a1ccc8a28937c7c671893d5fc-1 {}] regionserver.HStore(327): Store=5f1c5a6a1ccc8a28937c7c671893d5fc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:26:41,884 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,884 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,885 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,886 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,886 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,886 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:26:41,888 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,890 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:26:41,891 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 5f1c5a6a1ccc8a28937c7c671893d5fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65897497, jitterRate=-0.018050774931907654}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:26:41,891 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:41,892 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: Running coprocessor pre-open hook at 1731569201875Writing region info on filesystem at 1731569201875Initializing all the Stores at 1731569201876 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569201877 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569201877Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569201877Cleaning up temporary data from old regions at 1731569201886 (+9 ms)Running coprocessor post-open hooks at 1731569201891 (+5 ms)Region opened successfully at 1731569201891 2024-11-14T07:26:41,892 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., pid=126, masterSystemTime=1731569201866 2024-11-14T07:26:41,894 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,894 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:41,895 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=5f1c5a6a1ccc8a28937c7c671893d5fc, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:41,896 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42927 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=c7aa6e2790b3,46235,1731569085523, table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T07:26:41,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:26:41,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T07:26:41,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 in 186 msec 2024-11-14T07:26:41,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-14T07:26:41,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, ASSIGN in 344 msec 2024-11-14T07:26:41,901 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:26:41,901 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569201901"}]},"ts":"1731569201901"} 2024-11-14T07:26:41,903 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:26:41,903 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:26:41,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 773 msec 2024-11-14T07:26:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-14T07:26:42,275 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:42,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,278 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12473da4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44df2f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:26:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:26:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bdf3363 2024-11-14T07:26:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:26:42,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:26:42,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,290 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b97d9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23359105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf4c5b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,294 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bd0e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,296 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f85ba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,297 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c3f310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,299 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6b3160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c24af1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6096303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe2cfd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fcb40ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a61c835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364293d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497fd169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c03124c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813b8d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215c3b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a452f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bb5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:26:42,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:26:42,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fff6daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:26:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:26:42,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:26:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fcbb72b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:26:42,322 DEBUG [Thread-2201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 DEBUG [Thread-2203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:42,323 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,323 DEBUG [Thread-2203 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [Thread-2201 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,324 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,324 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,324 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:26:42,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T07:26:42,325 DEBUG [Thread-2203 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:26:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:26:42,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:26:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,330 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:42,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2] 2024-11-14T07:26:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:42,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:26:42,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c0b559a886eb4ba7811fc02cbb1d7e0b is 50, key is test_row_0/A:col10/1731569202337/Put/seqid=0 2024-11-14T07:26:42,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742321_1497 (size=14341) 2024-11-14T07:26:42,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c0b559a886eb4ba7811fc02cbb1d7e0b 2024-11-14T07:26:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569262364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569262365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569262374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569262375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569262376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/31c0a080542e4307b0a7f7ab035772a9 is 50, key is test_row_0/B:col10/1731569202337/Put/seqid=0 2024-11-14T07:26:42,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742322_1498 (size=12001) 2024-11-14T07:26:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:42,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:26:42,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569262484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569262484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569262484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569262494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569262495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:26:42,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569262694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569262694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569262694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569262705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569262705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:42,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:26:42,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:42,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:42,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/31c0a080542e4307b0a7f7ab035772a9 2024-11-14T07:26:42,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/43ce5075aeb0492b97479d9e91c92463 is 50, key is test_row_0/C:col10/1731569202337/Put/seqid=0 2024-11-14T07:26:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742323_1499 (size=12001) 2024-11-14T07:26:42,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/43ce5075aeb0492b97479d9e91c92463 2024-11-14T07:26:42,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c0b559a886eb4ba7811fc02cbb1d7e0b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b 2024-11-14T07:26:42,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b, entries=200, sequenceid=14, filesize=14.0 K 2024-11-14T07:26:42,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/31c0a080542e4307b0a7f7ab035772a9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9 2024-11-14T07:26:42,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T07:26:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/43ce5075aeb0492b97479d9e91c92463 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463 2024-11-14T07:26:42,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T07:26:42,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 490ms, sequenceid=14, compaction requested=false 2024-11-14T07:26:42,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:42,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T07:26:42,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:42,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:26:42,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/6890d9c49fa64a83b2ad77590625e647 is 50, key is test_row_0/A:col10/1731569202374/Put/seqid=0 2024-11-14T07:26:42,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742324_1500 (size=12001) 2024-11-14T07:26:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:43,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:43,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569263042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569263042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569263042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569263043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569263043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/6890d9c49fa64a83b2ad77590625e647 2024-11-14T07:26:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/1590a417f9d543f28c898846c80ac5d5 is 50, key is test_row_0/B:col10/1731569202374/Put/seqid=0 2024-11-14T07:26:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742325_1501 (size=12001) 2024-11-14T07:26:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569263364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569263364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569263364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569263364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569263364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:43,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569263675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569263675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569263675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569263675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569263675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:43,762 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/1590a417f9d543f28c898846c80ac5d5 2024-11-14T07:26:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a59d34b2e56a42b790f362f65ef324ce is 50, key is test_row_0/C:col10/1731569202374/Put/seqid=0 2024-11-14T07:26:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742326_1502 (size=12001) 2024-11-14T07:26:43,770 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a59d34b2e56a42b790f362f65ef324ce 2024-11-14T07:26:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/6890d9c49fa64a83b2ad77590625e647 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647 2024-11-14T07:26:43,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:26:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/1590a417f9d543f28c898846c80ac5d5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5 2024-11-14T07:26:43,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:26:43,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a59d34b2e56a42b790f362f65ef324ce as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce 2024-11-14T07:26:43,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T07:26:43,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 848ms, sequenceid=38, compaction requested=false 2024-11-14T07:26:43,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:43,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:43,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T07:26:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T07:26:43,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T07:26:43,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-14T07:26:43,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4690 sec 2024-11-14T07:26:43,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:26:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:44,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:26:44,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/b99405e5554d4cf7a227672d728389a3 is 50, key is test_row_0/A:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742327_1503 (size=12001) 2024-11-14T07:26:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569264216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569264219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569264220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569264225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569264226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569264334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569264335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569264335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569264344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569264344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T07:26:44,475 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:44,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T07:26:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:44,476 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:44,477 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:44,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569264544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569264544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569264554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569264554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569264554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:44,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/b99405e5554d4cf7a227672d728389a3 2024-11-14T07:26:44,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/841b6a07d4ed4492a897d6b57f06ebc2 is 50, key is test_row_0/B:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742328_1504 (size=12001) 2024-11-14T07:26:44,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:44,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569264854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569264854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569264864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569264864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569264864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:44,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:44,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/841b6a07d4ed4492a897d6b57f06ebc2 2024-11-14T07:26:45,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/b67f9e3dab9845aaa2bbdd0436e9aba7 is 50, key is test_row_0/C:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:45,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742329_1505 (size=12001) 2024-11-14T07:26:45,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:45,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:45,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:45,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569265364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569265364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:45,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569265374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569265385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569265385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:45,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:45,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:45,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:45,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/b67f9e3dab9845aaa2bbdd0436e9aba7 2024-11-14T07:26:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/b99405e5554d4cf7a227672d728389a3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3 2024-11-14T07:26:45,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T07:26:45,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/841b6a07d4ed4492a897d6b57f06ebc2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2 2024-11-14T07:26:45,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T07:26:45,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/b67f9e3dab9845aaa2bbdd0436e9aba7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7 2024-11-14T07:26:45,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T07:26:45,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1290ms, sequenceid=52, compaction requested=true 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:45,478 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:45,478 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:45,479 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,479 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,479 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.2 K 2024-11-14T07:26:45,479 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=37.4 K 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c0a080542e4307b0a7f7ab035772a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731569202334 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b559a886eb4ba7811fc02cbb1d7e0b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731569202331 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6890d9c49fa64a83b2ad77590625e647, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569202353 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1590a417f9d543f28c898846c80ac5d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569202353 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99405e5554d4cf7a227672d728389a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:45,479 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 841b6a07d4ed4492a897d6b57f06ebc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:45,484 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:45,485 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/747a529bf9124192a11b2b3e7d819dff is 50, key is test_row_0/B:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:45,487 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:45,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742330_1506 (size=12104) 2024-11-14T07:26:45,488 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a7138e73d0a546e584b7599cbe3b7593 is 50, key is test_row_0/A:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:45,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742331_1507 (size=12104) 2024-11-14T07:26:45,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T07:26:45,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T07:26:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7451bda7b2434f49b2039fad6d585a66 is 50, key is test_row_0/A:col10/1731569204206/Put/seqid=0 2024-11-14T07:26:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742332_1508 (size=12001) 2024-11-14T07:26:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:45,892 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/747a529bf9124192a11b2b3e7d819dff as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/747a529bf9124192a11b2b3e7d819dff 2024-11-14T07:26:45,896 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into 747a529bf9124192a11b2b3e7d819dff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:45,896 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a7138e73d0a546e584b7599cbe3b7593 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a7138e73d0a546e584b7599cbe3b7593 2024-11-14T07:26:45,896 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:45,896 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569205478; duration=0sec 2024-11-14T07:26:45,896 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:45,896 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:45,896 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:45,897 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:45,897 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:45,897 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:45,897 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.2 K 2024-11-14T07:26:45,897 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ce5075aeb0492b97479d9e91c92463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731569202334 2024-11-14T07:26:45,897 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a59d34b2e56a42b790f362f65ef324ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731569202353 2024-11-14T07:26:45,898 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b67f9e3dab9845aaa2bbdd0436e9aba7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:45,900 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into a7138e73d0a546e584b7599cbe3b7593(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:45,900 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:45,900 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569205478; duration=0sec 2024-11-14T07:26:45,900 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:45,900 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:45,905 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:45,905 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/483996e43a23474d99915cf4f21e42d0 is 50, key is test_row_0/C:col10/1731569204187/Put/seqid=0 2024-11-14T07:26:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742333_1509 (size=12104) 2024-11-14T07:26:45,912 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/483996e43a23474d99915cf4f21e42d0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/483996e43a23474d99915cf4f21e42d0 2024-11-14T07:26:45,915 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into 483996e43a23474d99915cf4f21e42d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:45,915 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:45,915 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569205478; duration=0sec 2024-11-14T07:26:45,915 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:45,915 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:45,953 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7451bda7b2434f49b2039fad6d585a66 2024-11-14T07:26:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/752f3b08f2284539b2974a14e829e2c5 is 50, key is test_row_0/B:col10/1731569204206/Put/seqid=0 2024-11-14T07:26:45,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742334_1510 (size=12001) 2024-11-14T07:26:45,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/752f3b08f2284539b2974a14e829e2c5 2024-11-14T07:26:45,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/05f0be5e685f4352825d5c401964859f is 50, key is test_row_0/C:col10/1731569204206/Put/seqid=0 2024-11-14T07:26:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742335_1511 (size=12001) 2024-11-14T07:26:46,306 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:26:46,371 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/05f0be5e685f4352825d5c401964859f 2024-11-14T07:26:46,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7451bda7b2434f49b2039fad6d585a66 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66 2024-11-14T07:26:46,377 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:26:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/752f3b08f2284539b2974a14e829e2c5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5 2024-11-14T07:26:46,381 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:26:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/05f0be5e685f4352825d5c401964859f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f 2024-11-14T07:26:46,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T07:26:46,386 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 841ms, sequenceid=74, compaction requested=false 2024-11-14T07:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T07:26:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T07:26:46,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T07:26:46,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-11-14T07:26:46,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9140 sec 2024-11-14T07:26:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:46,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:46,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f38dca9227cf4ec88fda095200911fca is 50, key is test_row_0/A:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:46,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742336_1512 (size=16681) 2024-11-14T07:26:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569266434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569266435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569266437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569266439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569266440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569266544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T07:26:46,624 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:46,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T07:26:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:46,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:46,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:46,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:46,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569266754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569266764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569266764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569266765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569266765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:46,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:46,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f38dca9227cf4ec88fda095200911fca 2024-11-14T07:26:46,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/df85310e008246b5a1a39b85f024e6f4 is 50, key is test_row_0/B:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742337_1513 (size=12001) 2024-11-14T07:26:46,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:46,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569267064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569267084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569267084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569267084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569267085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/df85310e008246b5a1a39b85f024e6f4 2024-11-14T07:26:47,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a672aceb9840450f8c475f734d551d4a is 50, key is test_row_0/C:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742338_1514 (size=12001) 2024-11-14T07:26:47,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:47,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:47,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569267574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569267594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569267595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569267595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569267604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:47,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a672aceb9840450f8c475f734d551d4a 2024-11-14T07:26:47,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f38dca9227cf4ec88fda095200911fca as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca 2024-11-14T07:26:47,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca, entries=250, sequenceid=88, filesize=16.3 K 2024-11-14T07:26:47,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/df85310e008246b5a1a39b85f024e6f4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4 2024-11-14T07:26:47,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T07:26:47,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a672aceb9840450f8c475f734d551d4a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a 2024-11-14T07:26:47,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T07:26:47,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1249ms, sequenceid=88, compaction requested=true 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:47,649 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:47,649 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,650 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,650 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/747a529bf9124192a11b2b3e7d819dff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.3 K 2024-11-14T07:26:47,650 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a7138e73d0a546e584b7599cbe3b7593, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=39.8 K 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7138e73d0a546e584b7599cbe3b7593, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 747a529bf9124192a11b2b3e7d819dff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7451bda7b2434f49b2039fad6d585a66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569204206 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 752f3b08f2284539b2974a14e829e2c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569204206 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting df85310e008246b5a1a39b85f024e6f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:47,650 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38dca9227cf4ec88fda095200911fca, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:47,656 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:47,656 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:47,657 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/fc5b862eaae44746becf414d14742559 is 50, key is test_row_0/B:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:47,657 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e1b803aed06440578b96b9dde45cad9d is 50, key is test_row_0/A:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742340_1516 (size=12207) 2024-11-14T07:26:47,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742339_1515 (size=12207) 2024-11-14T07:26:47,679 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/fc5b862eaae44746becf414d14742559 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/fc5b862eaae44746becf414d14742559 2024-11-14T07:26:47,679 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e1b803aed06440578b96b9dde45cad9d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e1b803aed06440578b96b9dde45cad9d 2024-11-14T07:26:47,684 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into e1b803aed06440578b96b9dde45cad9d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:47,684 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into fc5b862eaae44746becf414d14742559(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:47,684 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569207649; duration=0sec 2024-11-14T07:26:47,684 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569207649; duration=0sec 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:47,684 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:47,685 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:47,685 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:47,685 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,685 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/483996e43a23474d99915cf4f21e42d0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.3 K 2024-11-14T07:26:47,686 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 483996e43a23474d99915cf4f21e42d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731569203008 2024-11-14T07:26:47,687 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f0be5e685f4352825d5c401964859f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731569204206 2024-11-14T07:26:47,687 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a672aceb9840450f8c475f734d551d4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:47,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T07:26:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:47,694 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:47,695 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:47,696 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/d15a465385e7430aa78069505e1bc0a8 is 50, key is test_row_0/C:col10/1731569206395/Put/seqid=0 2024-11-14T07:26:47,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/648cf744e3484a0684e789bbae9a6833 is 50, key is test_row_0/A:col10/1731569206433/Put/seqid=0 2024-11-14T07:26:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742341_1517 (size=12207) 2024-11-14T07:26:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742342_1518 (size=12001) 2024-11-14T07:26:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:48,107 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/648cf744e3484a0684e789bbae9a6833 2024-11-14T07:26:48,109 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/d15a465385e7430aa78069505e1bc0a8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d15a465385e7430aa78069505e1bc0a8 2024-11-14T07:26:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/d9fae247487d4608a7010929f61c248e is 50, key is test_row_0/B:col10/1731569206433/Put/seqid=0 2024-11-14T07:26:48,117 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into d15a465385e7430aa78069505e1bc0a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:48,117 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:48,117 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569207649; duration=0sec 2024-11-14T07:26:48,117 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:48,117 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:48,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742343_1519 (size=12001) 2024-11-14T07:26:48,121 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/d9fae247487d4608a7010929f61c248e 2024-11-14T07:26:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/3040d309bfa04e46970d27176c684a69 is 50, key is test_row_0/C:col10/1731569206433/Put/seqid=0 2024-11-14T07:26:48,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742344_1520 (size=12001) 2024-11-14T07:26:48,136 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/3040d309bfa04e46970d27176c684a69 2024-11-14T07:26:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/648cf744e3484a0684e789bbae9a6833 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833 2024-11-14T07:26:48,147 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T07:26:48,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/d9fae247487d4608a7010929f61c248e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e 2024-11-14T07:26:48,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T07:26:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/3040d309bfa04e46970d27176c684a69 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69 2024-11-14T07:26:48,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T07:26:48,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 466ms, sequenceid=115, compaction requested=false 2024-11-14T07:26:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T07:26:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T07:26:48,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T07:26:48,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-11-14T07:26:48,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5390 sec 2024-11-14T07:26:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:48,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:48,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/8d43b6da42e54d30b44afca449414c7a is 50, key is test_row_0/A:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:48,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742345_1521 (size=11997) 2024-11-14T07:26:48,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569268640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569268648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569268649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569268652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569268652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569268754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569268764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569268765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569268765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569268765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T07:26:48,774 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:48,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T07:26:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:48,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:48,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:48,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:48,866 WARN [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-14T07:26:48,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a7138e73d0a546e584b7599cbe3b7593, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca] to archive 2024-11-14T07:26:48,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:48,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c0b559a886eb4ba7811fc02cbb1d7e0b 2024-11-14T07:26:48,870 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/6890d9c49fa64a83b2ad77590625e647 2024-11-14T07:26:48,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a7138e73d0a546e584b7599cbe3b7593 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a7138e73d0a546e584b7599cbe3b7593 2024-11-14T07:26:48,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/b99405e5554d4cf7a227672d728389a3 2024-11-14T07:26:48,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7451bda7b2434f49b2039fad6d585a66 2024-11-14T07:26:48,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f38dca9227cf4ec88fda095200911fca 2024-11-14T07:26:48,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/747a529bf9124192a11b2b3e7d819dff, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4] to archive 2024-11-14T07:26:48,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:48,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/31c0a080542e4307b0a7f7ab035772a9 2024-11-14T07:26:48,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1590a417f9d543f28c898846c80ac5d5 2024-11-14T07:26:48,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/747a529bf9124192a11b2b3e7d819dff to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/747a529bf9124192a11b2b3e7d819dff 2024-11-14T07:26:48,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/841b6a07d4ed4492a897d6b57f06ebc2 2024-11-14T07:26:48,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/752f3b08f2284539b2974a14e829e2c5 2024-11-14T07:26:48,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/df85310e008246b5a1a39b85f024e6f4 2024-11-14T07:26:48,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/483996e43a23474d99915cf4f21e42d0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a] to archive 2024-11-14T07:26:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:48,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:26:48,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/43ce5075aeb0492b97479d9e91c92463 2024-11-14T07:26:48,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a59d34b2e56a42b790f362f65ef324ce 2024-11-14T07:26:48,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/483996e43a23474d99915cf4f21e42d0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/483996e43a23474d99915cf4f21e42d0 2024-11-14T07:26:48,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/b67f9e3dab9845aaa2bbdd0436e9aba7 2024-11-14T07:26:48,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/05f0be5e685f4352825d5c401964859f 2024-11-14T07:26:48,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c7aa6e2790b3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a672aceb9840450f8c475f734d551d4a 2024-11-14T07:26:48,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:48,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:48,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:48,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:48,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:48,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:48,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569268964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569268975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569268975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569268975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569268975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:48,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/8d43b6da42e54d30b44afca449414c7a 2024-11-14T07:26:49,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 is 50, key is test_row_0/B:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:49,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742346_1522 (size=9657) 2024-11-14T07:26:49,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:49,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569269274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569269284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569269285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569269294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569269294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:49,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 2024-11-14T07:26:49,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/28999bab919341d59e04858fc59aac1c is 50, key is test_row_0/C:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:49,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742347_1523 (size=9657) 2024-11-14T07:26:49,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569269784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569269794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569269794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569269804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569269804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:49,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/28999bab919341d59e04858fc59aac1c 2024-11-14T07:26:49,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:49,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:49,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/8d43b6da42e54d30b44afca449414c7a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a 2024-11-14T07:26:49,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T07:26:49,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 2024-11-14T07:26:49,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85, entries=100, sequenceid=127, filesize=9.4 K 2024-11-14T07:26:49,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/28999bab919341d59e04858fc59aac1c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c 2024-11-14T07:26:49,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c, entries=100, sequenceid=127, filesize=9.4 K 2024-11-14T07:26:49,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1248ms, sequenceid=127, compaction requested=true 2024-11-14T07:26:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:49,855 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:49,856 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:49,856 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:49,856 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:49,856 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:49,856 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:49,856 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,856 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,856 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e1b803aed06440578b96b9dde45cad9d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.4 K 2024-11-14T07:26:49,856 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/fc5b862eaae44746becf414d14742559, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=33.1 K 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1b803aed06440578b96b9dde45cad9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fc5b862eaae44746becf414d14742559, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 648cf744e3484a0684e789bbae9a6833, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731569206433 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d9fae247487d4608a7010929f61c248e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731569206433 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c2ad1a8fe04e04baaa9e3bc7b7ff85, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569208601 2024-11-14T07:26:49,857 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d43b6da42e54d30b44afca449414c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569208591 2024-11-14T07:26:49,862 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:49,862 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/db883680972a436a8ae677769a70bdcf is 50, key is test_row_0/A:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:49,864 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:49,865 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/7c203f727c764c16b7114e87ce830f53 is 50, key is test_row_0/B:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742349_1525 (size=12104) 2024-11-14T07:26:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742348_1524 (size=12104) 2024-11-14T07:26:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:49,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T07:26:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:49,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:50,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5c8ab1b758654d909370146c5d5f1fc0 is 50, key is test_row_0/A:col10/1731569208648/Put/seqid=0 2024-11-14T07:26:50,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742350_1526 (size=12151) 2024-11-14T07:26:50,273 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/db883680972a436a8ae677769a70bdcf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/db883680972a436a8ae677769a70bdcf 2024-11-14T07:26:50,273 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/7c203f727c764c16b7114e87ce830f53 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/7c203f727c764c16b7114e87ce830f53 2024-11-14T07:26:50,277 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into 7c203f727c764c16b7114e87ce830f53(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:50,277 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569209856; duration=0sec 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:50,277 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into db883680972a436a8ae677769a70bdcf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:50,277 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569209855; duration=0sec 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:50,277 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:50,278 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:50,278 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:50,278 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:50,278 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d15a465385e7430aa78069505e1bc0a8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=33.1 K 2024-11-14T07:26:50,278 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d15a465385e7430aa78069505e1bc0a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731569206391 2024-11-14T07:26:50,278 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 3040d309bfa04e46970d27176c684a69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731569206433 2024-11-14T07:26:50,279 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 28999bab919341d59e04858fc59aac1c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569208601 2024-11-14T07:26:50,284 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:50,285 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/1f3e8b9b128f4767bef46528a0b28f33 is 50, key is test_row_0/C:col10/1731569208601/Put/seqid=0 2024-11-14T07:26:50,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742351_1527 (size=12104) 2024-11-14T07:26:50,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5c8ab1b758654d909370146c5d5f1fc0 2024-11-14T07:26:50,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2bc338de23834b408f769c6fbe95d6ae is 50, key is test_row_0/B:col10/1731569208648/Put/seqid=0 2024-11-14T07:26:50,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742352_1528 (size=12151) 2024-11-14T07:26:50,741 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/1f3e8b9b128f4767bef46528a0b28f33 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/1f3e8b9b128f4767bef46528a0b28f33 2024-11-14T07:26:50,745 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into 1f3e8b9b128f4767bef46528a0b28f33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:50,745 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:50,745 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569209856; duration=0sec 2024-11-14T07:26:50,745 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:50,745 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:50,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2bc338de23834b408f769c6fbe95d6ae 2024-11-14T07:26:50,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bc8d6e181b914dce807e1da2e1a245f7 is 50, key is test_row_0/C:col10/1731569208648/Put/seqid=0 2024-11-14T07:26:50,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742353_1529 (size=12151) 2024-11-14T07:26:50,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bc8d6e181b914dce807e1da2e1a245f7 2024-11-14T07:26:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5c8ab1b758654d909370146c5d5f1fc0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0 2024-11-14T07:26:50,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569270824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:50,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T07:26:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2bc338de23834b408f769c6fbe95d6ae as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae 2024-11-14T07:26:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T07:26:50,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bc8d6e181b914dce807e1da2e1a245f7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7 2024-11-14T07:26:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569270825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569270825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569270825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:50,833 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:50,833 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569270826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:50,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T07:26:50,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 841ms, sequenceid=152, compaction requested=false 2024-11-14T07:26:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T07:26:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T07:26:50,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T07:26:50,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0610 sec 2024-11-14T07:26:50,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0660 sec 2024-11-14T07:26:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T07:26:50,924 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:50,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-14T07:26:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:50,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:50,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:50,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:50,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:26:50,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5487bc09cdca418292568dcaada01acd is 50, key is test_row_0/A:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742354_1530 (size=14541) 2024-11-14T07:26:51,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569271001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569271009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569271009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:51,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569271114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569271124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569271124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:51,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569271324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569271334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569271334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5487bc09cdca418292568dcaada01acd 2024-11-14T07:26:51,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/de58d0368bdb4d9791054825d28fdb71 is 50, key is test_row_0/B:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:51,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742355_1531 (size=12151) 2024-11-14T07:26:51,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:51,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569271635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569271644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569271644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:51,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/de58d0368bdb4d9791054825d28fdb71 2024-11-14T07:26:51,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/c1bfdfcd895e40c2a9a357ca55aa826f is 50, key is test_row_0/C:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:51,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742356_1532 (size=12151) 2024-11-14T07:26:51,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:51,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:52,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569272144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:52,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:52,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569272154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569272165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/c1bfdfcd895e40c2a9a357ca55aa826f 2024-11-14T07:26:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/5487bc09cdca418292568dcaada01acd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd 2024-11-14T07:26:52,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd, entries=200, sequenceid=167, filesize=14.2 K 2024-11-14T07:26:52,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/de58d0368bdb4d9791054825d28fdb71 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71 2024-11-14T07:26:52,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T07:26:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/c1bfdfcd895e40c2a9a357ca55aa826f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f 2024-11-14T07:26:52,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T07:26:52,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1245ms, sequenceid=167, compaction requested=true 2024-11-14T07:26:52,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38796 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:52,185 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:52,185 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,185 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/db883680972a436a8ae677769a70bdcf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=37.9 K 2024-11-14T07:26:52,186 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/7c203f727c764c16b7114e87ce830f53, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.6 K 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting db883680972a436a8ae677769a70bdcf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569206439 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c203f727c764c16b7114e87ce830f53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569206439 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc338de23834b408f769c6fbe95d6ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731569208647 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c8ab1b758654d909370146c5d5f1fc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731569208647 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting de58d0368bdb4d9791054825d28fdb71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:52,186 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5487bc09cdca418292568dcaada01acd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:52,193 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:52,193 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:52,194 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/fe720b4a718f463eb72467a27060625c is 50, key is test_row_0/A:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:52,194 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2dba4e8b229d49a296cc2ab99685ee49 is 50, key is test_row_0/B:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:52,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742357_1533 (size=12357) 2024-11-14T07:26:52,202 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/fe720b4a718f463eb72467a27060625c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/fe720b4a718f463eb72467a27060625c 2024-11-14T07:26:52,206 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into fe720b4a718f463eb72467a27060625c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:52,206 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:52,206 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569212185; duration=0sec 2024-11-14T07:26:52,206 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:52,206 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:52,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742358_1534 (size=12357) 2024-11-14T07:26:52,206 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:52,207 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:52,207 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:52,207 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,207 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/1f3e8b9b128f4767bef46528a0b28f33, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.6 K 2024-11-14T07:26:52,207 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f3e8b9b128f4767bef46528a0b28f33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731569206439 2024-11-14T07:26:52,208 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8d6e181b914dce807e1da2e1a245f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731569208647 2024-11-14T07:26:52,208 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1bfdfcd895e40c2a9a357ca55aa826f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:52,213 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:52,214 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a412a1b001a5403582b4cb01910ba484 is 50, key is test_row_0/C:col10/1731569210824/Put/seqid=0 2024-11-14T07:26:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742359_1535 (size=12357) 2024-11-14T07:26:52,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T07:26:52,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:52,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:52,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e4b4513c91854126b12b9fe24d700dda is 50, key is test_row_0/A:col10/1731569211007/Put/seqid=0 2024-11-14T07:26:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742360_1536 (size=12151) 2024-11-14T07:26:52,610 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2dba4e8b229d49a296cc2ab99685ee49 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2dba4e8b229d49a296cc2ab99685ee49 2024-11-14T07:26:52,614 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into 2dba4e8b229d49a296cc2ab99685ee49(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:52,614 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:52,614 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569212185; duration=0sec 2024-11-14T07:26:52,614 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:52,614 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:52,625 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a412a1b001a5403582b4cb01910ba484 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a412a1b001a5403582b4cb01910ba484 2024-11-14T07:26:52,628 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into a412a1b001a5403582b4cb01910ba484(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:52,628 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:52,629 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569212185; duration=0sec 2024-11-14T07:26:52,629 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:52,629 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:52,708 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e4b4513c91854126b12b9fe24d700dda 2024-11-14T07:26:52,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/4a324bead5c54cf18d2592423845c519 is 50, key is test_row_0/B:col10/1731569211007/Put/seqid=0 2024-11-14T07:26:52,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742361_1537 (size=12151) 2024-11-14T07:26:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:52,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:52,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569272866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569272870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569272975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:52,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569272984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:53,119 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/4a324bead5c54cf18d2592423845c519 2024-11-14T07:26:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5f3421c295c548d1a18a3a7f29417cc8 is 50, key is test_row_0/C:col10/1731569211007/Put/seqid=0 2024-11-14T07:26:53,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742362_1538 (size=12151) 2024-11-14T07:26:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569273154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,159 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569273164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,170 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569273185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569273185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,188 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569273194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569273494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569273505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:53,529 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5f3421c295c548d1a18a3a7f29417cc8 2024-11-14T07:26:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e4b4513c91854126b12b9fe24d700dda as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda 2024-11-14T07:26:53,535 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T07:26:53,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/4a324bead5c54cf18d2592423845c519 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519 2024-11-14T07:26:53,538 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T07:26:53,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5f3421c295c548d1a18a3a7f29417cc8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8 2024-11-14T07:26:53,541 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-14T07:26:53,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1241ms, sequenceid=193, compaction requested=false 2024-11-14T07:26:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-14T07:26:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-14T07:26:53,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T07:26:53,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-11-14T07:26:53,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.6200 sec 2024-11-14T07:26:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:54,010 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:54,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:26:54,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/222cfcd3d9be47a6852fe1fc329e0daf is 50, key is test_row_0/A:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742363_1539 (size=14541) 2024-11-14T07:26:54,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/222cfcd3d9be47a6852fe1fc329e0daf 2024-11-14T07:26:54,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/23f153ab0b6946a9b8572b44a195c91e is 50, key is test_row_0/B:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742364_1540 (size=12151) 2024-11-14T07:26:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569274103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569274104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569274215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569274215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569274424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569274424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/23f153ab0b6946a9b8572b44a195c91e 2024-11-14T07:26:54,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bf0927e1fb1b49ee9a0860da640c1d14 is 50, key is test_row_0/C:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742365_1541 (size=12151) 2024-11-14T07:26:54,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bf0927e1fb1b49ee9a0860da640c1d14 2024-11-14T07:26:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/222cfcd3d9be47a6852fe1fc329e0daf as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf 2024-11-14T07:26:54,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf, entries=200, sequenceid=207, filesize=14.2 K 2024-11-14T07:26:54,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/23f153ab0b6946a9b8572b44a195c91e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e 2024-11-14T07:26:54,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T07:26:54,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/bf0927e1fb1b49ee9a0860da640c1d14 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14 2024-11-14T07:26:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T07:26:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 447ms, sequenceid=207, compaction requested=true 2024-11-14T07:26:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:54,458 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:54,458 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:54,459 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:54,459 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:54,459 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/fe720b4a718f463eb72467a27060625c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=38.1 K 2024-11-14T07:26:54,459 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2dba4e8b229d49a296cc2ab99685ee49, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.8 K 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dba4e8b229d49a296cc2ab99685ee49, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:54,459 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe720b4a718f463eb72467a27060625c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:54,460 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b4513c91854126b12b9fe24d700dda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731569210958 2024-11-14T07:26:54,460 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a324bead5c54cf18d2592423845c519, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731569210958 2024-11-14T07:26:54,460 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222cfcd3d9be47a6852fe1fc329e0daf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:54,460 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f153ab0b6946a9b8572b44a195c91e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:54,465 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:54,465 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/2d81a04a258d4d758206d932e8dcf846 is 50, key is test_row_0/A:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,466 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:54,466 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/cb798d98359e4a10b985dc939463da55 is 50, key is test_row_0/B:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742366_1542 (size=12459) 2024-11-14T07:26:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742367_1543 (size=12459) 2024-11-14T07:26:54,473 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/2d81a04a258d4d758206d932e8dcf846 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/2d81a04a258d4d758206d932e8dcf846 2024-11-14T07:26:54,478 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into 2d81a04a258d4d758206d932e8dcf846(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:54,478 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:54,478 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569214457; duration=0sec 2024-11-14T07:26:54,478 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:54,478 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:54,478 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:54,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:54,479 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:54,479 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:54,479 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a412a1b001a5403582b4cb01910ba484, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=35.8 K 2024-11-14T07:26:54,480 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a412a1b001a5403582b4cb01910ba484, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731569210823 2024-11-14T07:26:54,480 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f3421c295c548d1a18a3a7f29417cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731569210958 2024-11-14T07:26:54,480 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0927e1fb1b49ee9a0860da640c1d14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:54,486 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:54,486 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a2700b8b58314d98925887905b520d37 is 50, key is test_row_0/C:col10/1731569212865/Put/seqid=0 2024-11-14T07:26:54,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742368_1544 (size=12459) 2024-11-14T07:26:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:54,739 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:54,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:54,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a1627ac908704d5faeeb2b1d38d26778 is 50, key is test_row_0/A:col10/1731569214739/Put/seqid=0 2024-11-14T07:26:54,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742369_1545 (size=16931) 2024-11-14T07:26:54,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569274770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569274779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,873 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/cb798d98359e4a10b985dc939463da55 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/cb798d98359e4a10b985dc939463da55 2024-11-14T07:26:54,877 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into cb798d98359e4a10b985dc939463da55(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:54,877 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:54,878 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569214458; duration=0sec 2024-11-14T07:26:54,878 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:54,878 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569274884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:54,894 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/a2700b8b58314d98925887905b520d37 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a2700b8b58314d98925887905b520d37 2024-11-14T07:26:54,898 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into a2700b8b58314d98925887905b520d37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:54,898 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:54,898 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569214458; duration=0sec 2024-11-14T07:26:54,898 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:54,898 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:54,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569274894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T07:26:55,095 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:55,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-14T07:26:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:55,097 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:55,098 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:55,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:55,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569275104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569275114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a1627ac908704d5faeeb2b1d38d26778 2024-11-14T07:26:55,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/25f659123edf421983cb10bece4bdc7d is 50, key is test_row_0/B:col10/1731569214739/Put/seqid=0 2024-11-14T07:26:55,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742370_1546 (size=12151) 2024-11-14T07:26:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569275185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569275185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,190 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:55,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569275204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,207 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:55,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569275414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569275424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/25f659123edf421983cb10bece4bdc7d 2024-11-14T07:26:55,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/12a24055715f4d7b9cbdd3f520537d0c is 50, key is test_row_0/C:col10/1731569214739/Put/seqid=0 2024-11-14T07:26:55,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742371_1547 (size=12151) 2024-11-14T07:26:55,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:55,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:55,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:55,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569275924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569275934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:55,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/12a24055715f4d7b9cbdd3f520537d0c 2024-11-14T07:26:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a1627ac908704d5faeeb2b1d38d26778 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778 2024-11-14T07:26:55,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778, entries=250, sequenceid=234, filesize=16.5 K 2024-11-14T07:26:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/25f659123edf421983cb10bece4bdc7d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d 2024-11-14T07:26:55,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:26:55,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/12a24055715f4d7b9cbdd3f520537d0c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c 2024-11-14T07:26:55,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T07:26:55,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1249ms, sequenceid=234, compaction requested=false 2024-11-14T07:26:55,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:56,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-14T07:26:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:56,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T07:26:56,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7a2e6aee80ac4a35ad40372e9dd7163b is 50, key is test_row_0/A:col10/1731569214777/Put/seqid=0 2024-11-14T07:26:56,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742372_1548 (size=12151) 2024-11-14T07:26:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:56,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7a2e6aee80ac4a35ad40372e9dd7163b 2024-11-14T07:26:56,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/9e290d174eff455cb92d9bbe0a768de4 is 50, key is test_row_0/B:col10/1731569214777/Put/seqid=0 2024-11-14T07:26:56,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742373_1549 (size=12151) 2024-11-14T07:26:56,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/9e290d174eff455cb92d9bbe0a768de4 2024-11-14T07:26:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6b75e30773ea4abdb011e4c3b165ab19 is 50, key is test_row_0/C:col10/1731569214777/Put/seqid=0 2024-11-14T07:26:56,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742374_1550 (size=12151) 2024-11-14T07:26:56,464 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6b75e30773ea4abdb011e4c3b165ab19 2024-11-14T07:26:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7a2e6aee80ac4a35ad40372e9dd7163b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b 2024-11-14T07:26:56,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T07:26:56,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/9e290d174eff455cb92d9bbe0a768de4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4 2024-11-14T07:26:56,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T07:26:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6b75e30773ea4abdb011e4c3b165ab19 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19 2024-11-14T07:26:56,485 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T07:26:56,487 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 473ms, sequenceid=246, compaction requested=true 2024-11-14T07:26:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-14T07:26:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-14T07:26:56,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T07:26:56,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-11-14T07:26:56,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3950 sec 2024-11-14T07:26:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:56,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:26:56,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e82e642d7e2745a99976fb2185d6a113 is 50, key is test_row_0/A:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:56,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742375_1551 (size=14591) 2024-11-14T07:26:57,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569277085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569277097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569277205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569277215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T07:26:57,244 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:57,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-14T07:26:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:57,247 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:57,247 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:57,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:57,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e82e642d7e2745a99976fb2185d6a113 2024-11-14T07:26:57,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ebfe9b7250fd49c09940bf748b662640 is 50, key is test_row_0/B:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:57,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-14T07:26:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742376_1552 (size=12201) 2024-11-14T07:26:57,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569277424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569277434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-14T07:26:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:57,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:57,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-14T07:26:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569277745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569277754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:57,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ebfe9b7250fd49c09940bf748b662640 2024-11-14T07:26:57,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/152d2c987bd14a77b6dddd75c81d8f65 is 50, key is test_row_0/C:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:57,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742377_1553 (size=12201) 2024-11-14T07:26:57,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-14T07:26:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/152d2c987bd14a77b6dddd75c81d8f65 2024-11-14T07:26:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:57,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/e82e642d7e2745a99976fb2185d6a113 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113 2024-11-14T07:26:57,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113, entries=200, sequenceid=257, filesize=14.2 K 2024-11-14T07:26:57,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ebfe9b7250fd49c09940bf748b662640 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640 2024-11-14T07:26:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:57,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T07:26:57,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/152d2c987bd14a77b6dddd75c81d8f65 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65 2024-11-14T07:26:57,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T07:26:57,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 917ms, sequenceid=257, compaction requested=true 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:57,887 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:57,887 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:57,888 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:57,888 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:57,888 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,888 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/2d81a04a258d4d758206d932e8dcf846, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=54.8 K 2024-11-14T07:26:57,888 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d81a04a258d4d758206d932e8dcf846, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:57,889 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48962 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:57,889 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:57,889 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:57,889 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/cb798d98359e4a10b985dc939463da55, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=47.8 K 2024-11-14T07:26:57,889 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1627ac908704d5faeeb2b1d38d26778, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569214090 2024-11-14T07:26:57,889 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting cb798d98359e4a10b985dc939463da55, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:57,889 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a2e6aee80ac4a35ad40372e9dd7163b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731569214759 2024-11-14T07:26:57,890 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f659123edf421983cb10bece4bdc7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569214102 2024-11-14T07:26:57,890 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82e642d7e2745a99976fb2185d6a113, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:57,891 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e290d174eff455cb92d9bbe0a768de4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731569214759 2024-11-14T07:26:57,892 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfe9b7250fd49c09940bf748b662640, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:57,912 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:57,912 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/1b3ff684339a4758b6036438fc7330f2 is 50, key is test_row_0/B:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:57,933 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:57,934 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/598aa62cb0f245a08bf9c5c4e880c46c is 50, key is test_row_0/A:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:57,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742378_1554 (size=12645) 2024-11-14T07:26:58,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742379_1555 (size=12645) 2024-11-14T07:26:58,006 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/598aa62cb0f245a08bf9c5c4e880c46c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/598aa62cb0f245a08bf9c5c4e880c46c 2024-11-14T07:26:58,010 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into 598aa62cb0f245a08bf9c5c4e880c46c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:58,011 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:58,011 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=12, startTime=1731569217887; duration=0sec 2024-11-14T07:26:58,011 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:58,011 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:58,011 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:26:58,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-14T07:26:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:58,012 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:58,012 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48962 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:26:58,012 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:58,012 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:58,012 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a2700b8b58314d98925887905b520d37, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=47.8 K 2024-11-14T07:26:58,012 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2700b8b58314d98925887905b520d37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731569212860 2024-11-14T07:26:58,013 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a24055715f4d7b9cbdd3f520537d0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731569214102 2024-11-14T07:26:58,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b75e30773ea4abdb011e4c3b165ab19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731569214759 2024-11-14T07:26:58,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152d2c987bd14a77b6dddd75c81d8f65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/08d64f96b1d84a1891fc7d85205b9c09 is 50, key is test_row_0/A:col10/1731569217096/Put/seqid=0 2024-11-14T07:26:58,022 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:58,022 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/747540e6428942cd82d02d64d3826c91 is 50, key is test_row_0/C:col10/1731569216951/Put/seqid=0 2024-11-14T07:26:58,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742380_1556 (size=12301) 2024-11-14T07:26:58,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742381_1557 (size=12645) 2024-11-14T07:26:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:58,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569278308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569278308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,374 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/1b3ff684339a4758b6036438fc7330f2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1b3ff684339a4758b6036438fc7330f2 2024-11-14T07:26:58,377 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into 1b3ff684339a4758b6036438fc7330f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:58,377 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:58,378 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=12, startTime=1731569217887; duration=0sec 2024-11-14T07:26:58,378 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:58,378 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:58,426 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/08d64f96b1d84a1891fc7d85205b9c09 2024-11-14T07:26:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569278424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569278424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,432 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/747540e6428942cd82d02d64d3826c91 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/747540e6428942cd82d02d64d3826c91 2024-11-14T07:26:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/967961d533874dfca67ed30fed2113c7 is 50, key is test_row_0/B:col10/1731569217096/Put/seqid=0 2024-11-14T07:26:58,439 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into 747540e6428942cd82d02d64d3826c91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:58,439 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:58,439 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=12, startTime=1731569217887; duration=0sec 2024-11-14T07:26:58,439 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:58,440 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:58,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742382_1558 (size=12301) 2024-11-14T07:26:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569278634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569278634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:58,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/967961d533874dfca67ed30fed2113c7 2024-11-14T07:26:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/eeace6d854104404acf62fa41d939e25 is 50, key is test_row_0/C:col10/1731569217096/Put/seqid=0 2024-11-14T07:26:58,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742383_1559 (size=12301) 2024-11-14T07:26:58,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/eeace6d854104404acf62fa41d939e25 2024-11-14T07:26:58,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/08d64f96b1d84a1891fc7d85205b9c09 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09 2024-11-14T07:26:58,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09, entries=150, sequenceid=283, filesize=12.0 K 2024-11-14T07:26:58,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/967961d533874dfca67ed30fed2113c7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7 2024-11-14T07:26:58,888 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7, entries=150, sequenceid=283, filesize=12.0 K 2024-11-14T07:26:58,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/eeace6d854104404acf62fa41d939e25 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25 2024-11-14T07:26:58,894 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25, entries=150, sequenceid=283, filesize=12.0 K 2024-11-14T07:26:58,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 883ms, sequenceid=283, compaction requested=false 2024-11-14T07:26:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-14T07:26:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-14T07:26:58,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T07:26:58,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6520 sec 2024-11-14T07:26:58,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.6600 sec 2024-11-14T07:26:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:58,959 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:58,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:26:58,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c5d4e52fa0ab46be95324270217e3fe8 is 50, key is test_row_0/A:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:58,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742384_1560 (size=14741) 2024-11-14T07:26:58,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c5d4e52fa0ab46be95324270217e3fe8 2024-11-14T07:26:58,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/e9129f4f491c4252a4834bddf29946b8 is 50, key is test_row_0/B:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:59,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742385_1561 (size=12301) 2024-11-14T07:26:59,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/e9129f4f491c4252a4834bddf29946b8 2024-11-14T07:26:59,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/937da354550b4dda8e0fc942ef811f58 is 50, key is test_row_0/C:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:59,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742386_1562 (size=12301) 2024-11-14T07:26:59,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/937da354550b4dda8e0fc942ef811f58 2024-11-14T07:26:59,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c5d4e52fa0ab46be95324270217e3fe8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8 2024-11-14T07:26:59,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8, entries=200, sequenceid=297, filesize=14.4 K 2024-11-14T07:26:59,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/e9129f4f491c4252a4834bddf29946b8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8 2024-11-14T07:26:59,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T07:26:59,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/937da354550b4dda8e0fc942ef811f58 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58 2024-11-14T07:26:59,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T07:26:59,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 105ms, sequenceid=297, compaction requested=true 2024-11-14T07:26:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:26:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:59,064 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:26:59,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:59,064 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:26:59,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:59,065 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:59,065 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:59,065 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:26:59,065 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:26:59,065 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,065 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,065 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/598aa62cb0f245a08bf9c5c4e880c46c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=38.8 K 2024-11-14T07:26:59,065 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1b3ff684339a4758b6036438fc7330f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=36.4 K 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b3ff684339a4758b6036438fc7330f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598aa62cb0f245a08bf9c5c4e880c46c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08d64f96b1d84a1891fc7d85205b9c09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731569217068 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 967961d533874dfca67ed30fed2113c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731569217068 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e9129f4f491c4252a4834bddf29946b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:26:59,066 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d4e52fa0ab46be95324270217e3fe8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:26:59,075 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:59,076 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/d9e4deddd0de4784bb20d71b00f73a7f is 50, key is test_row_0/A:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:26:59,076 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:26:59,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:26:59,078 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:59,079 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/e35d47d4628f459492dc56fc03862bf2 is 50, key is test_row_0/B:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:59,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/9ca85c7fe3c745febdeb552790e2c103 is 50, key is test_row_0/A:col10/1731569219057/Put/seqid=0 2024-11-14T07:26:59,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742387_1563 (size=12847) 2024-11-14T07:26:59,102 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/d9e4deddd0de4784bb20d71b00f73a7f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/d9e4deddd0de4784bb20d71b00f73a7f 2024-11-14T07:26:59,106 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into d9e4deddd0de4784bb20d71b00f73a7f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:59,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:59,106 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=13, startTime=1731569219064; duration=0sec 2024-11-14T07:26:59,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:26:59,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:26:59,106 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:26:59,107 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:26:59,107 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:26:59,107 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,107 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/747540e6428942cd82d02d64d3826c91, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=36.4 K 2024-11-14T07:26:59,107 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747540e6428942cd82d02d64d3826c91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731569216951 2024-11-14T07:26:59,107 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeace6d854104404acf62fa41d939e25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731569217068 2024-11-14T07:26:59,108 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937da354550b4dda8e0fc942ef811f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:26:59,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742388_1564 (size=12847) 2024-11-14T07:26:59,128 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:26:59,129 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5d13374b6c3e4b96b632c4d7ccc7210d is 50, key is test_row_0/C:col10/1731569218295/Put/seqid=0 2024-11-14T07:26:59,134 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/e35d47d4628f459492dc56fc03862bf2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e35d47d4628f459492dc56fc03862bf2 2024-11-14T07:26:59,138 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into e35d47d4628f459492dc56fc03862bf2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:59,138 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:59,138 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=13, startTime=1731569219064; duration=0sec 2024-11-14T07:26:59,139 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:59,139 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:26:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569279131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569279131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742390_1566 (size=12847) 2024-11-14T07:26:59,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742389_1565 (size=14741) 2024-11-14T07:26:59,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/9ca85c7fe3c745febdeb552790e2c103 2024-11-14T07:26:59,155 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5d13374b6c3e4b96b632c4d7ccc7210d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5d13374b6c3e4b96b632c4d7ccc7210d 2024-11-14T07:26:59,160 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into 5d13374b6c3e4b96b632c4d7ccc7210d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:26:59,160 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:26:59,160 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=13, startTime=1731569219064; duration=0sec 2024-11-14T07:26:59,160 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:26:59,160 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:26:59,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/00c17075f01944e09f8c11e4c3f9bdbe is 50, key is test_row_0/B:col10/1731569219057/Put/seqid=0 2024-11-14T07:26:59,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742391_1567 (size=12301) 2024-11-14T07:26:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44546 deadline: 1731569279224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731569279225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,232 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,233 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44576 deadline: 1731569279235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,245 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 5f1c5a6a1ccc8a28937c7c671893d5fc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:26:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569279244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569279245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T07:26:59,394 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:26:59,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:26:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-14T07:26:59,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:26:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:26:59,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:26:59,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:26:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569279455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569279455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:26:59,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-14T07:26:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/00c17075f01944e09f8c11e4c3f9bdbe 2024-11-14T07:26:59,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/276aaed1d2cc468183e79d4247353d14 is 50, key is test_row_0/C:col10/1731569219057/Put/seqid=0 2024-11-14T07:26:59,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742392_1568 (size=12301) 2024-11-14T07:26:59,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-14T07:26:59,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:59,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:26:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569279775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:26:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569279775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:26:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:26:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:26:59,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-14T07:26:59,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:26:59,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:26:59,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:26:59,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/276aaed1d2cc468183e79d4247353d14 2024-11-14T07:27:00,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/9ca85c7fe3c745febdeb552790e2c103 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103 2024-11-14T07:27:00,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103, entries=200, sequenceid=322, filesize=14.4 K 2024-11-14T07:27:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/00c17075f01944e09f8c11e4c3f9bdbe as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe 2024-11-14T07:27:00,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-14T07:27:00,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:00,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:00,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:00,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:00,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe, entries=150, sequenceid=322, filesize=12.0 K 2024-11-14T07:27:00,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/276aaed1d2cc468183e79d4247353d14 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14 2024-11-14T07:27:00,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14, entries=150, sequenceid=322, filesize=12.0 K 2024-11-14T07:27:00,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 944ms, sequenceid=322, compaction requested=false 2024-11-14T07:27:00,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:27:00,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-14T07:27:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:00,162 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:27:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c1711da475c8496dade23820b5c53abc is 50, key is test_row_0/A:col10/1731569219089/Put/seqid=0 2024-11-14T07:27:00,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742393_1569 (size=12301) 2024-11-14T07:27:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c1711da475c8496dade23820b5c53abc 2024-11-14T07:27:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ed308af2e8274f47932c06e79f9db901 is 50, key is test_row_0/B:col10/1731569219089/Put/seqid=0 2024-11-14T07:27:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742394_1570 (size=12301) 2024-11-14T07:27:00,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ed308af2e8274f47932c06e79f9db901 2024-11-14T07:27:00,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/88c8f12f080f4d94b840c8e6147c9265 is 50, key is test_row_0/C:col10/1731569219089/Put/seqid=0 2024-11-14T07:27:00,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742395_1571 (size=12301) 2024-11-14T07:27:00,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:00,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569280392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569280400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569280505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569280515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:27:00,617 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/88c8f12f080f4d94b840c8e6147c9265 2024-11-14T07:27:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/c1711da475c8496dade23820b5c53abc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc 2024-11-14T07:27:00,625 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc, entries=150, sequenceid=337, filesize=12.0 K 2024-11-14T07:27:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ed308af2e8274f47932c06e79f9db901 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901 2024-11-14T07:27:00,629 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901, entries=150, sequenceid=337, filesize=12.0 K 2024-11-14T07:27:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/88c8f12f080f4d94b840c8e6147c9265 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265 2024-11-14T07:27:00,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265, entries=150, sequenceid=337, filesize=12.0 K 2024-11-14T07:27:00,634 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 472ms, sequenceid=337, compaction requested=true 2024-11-14T07:27:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-14T07:27:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-14T07:27:00,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T07:27:00,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2450 sec 2024-11-14T07:27:00,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.2530 sec 2024-11-14T07:27:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:00,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:00,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:27:00,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7052d2fc3fcf4a9aab85d895284841b5 is 50, key is test_row_0/A:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:00,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742396_1572 (size=14741) 2024-11-14T07:27:00,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7052d2fc3fcf4a9aab85d895284841b5 2024-11-14T07:27:00,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2442f4b63fdd46f2bd03caa44ba29244 is 50, key is test_row_0/B:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:00,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742397_1573 (size=12301) 2024-11-14T07:27:00,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2442f4b63fdd46f2bd03caa44ba29244 2024-11-14T07:27:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569280768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5dd1f4626b964a4bba850b95aefcaf60 is 50, key is test_row_0/C:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569280777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742398_1574 (size=12301) 2024-11-14T07:27:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569280894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569280894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569281105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569281114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5dd1f4626b964a4bba850b95aefcaf60 2024-11-14T07:27:01,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/7052d2fc3fcf4a9aab85d895284841b5 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5 2024-11-14T07:27:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5, entries=200, sequenceid=362, filesize=14.4 K 2024-11-14T07:27:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/2442f4b63fdd46f2bd03caa44ba29244 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244 2024-11-14T07:27:01,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244, entries=150, sequenceid=362, filesize=12.0 K 2024-11-14T07:27:01,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/5dd1f4626b964a4bba850b95aefcaf60 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60 2024-11-14T07:27:01,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60, entries=150, sequenceid=362, filesize=12.0 K 2024-11-14T07:27:01,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 497ms, sequenceid=362, compaction requested=true 2024-11-14T07:27:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:01,226 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:01,227 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f1c5a6a1ccc8a28937c7c671893d5fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:01,228 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:01,228 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/B is initiating minor compaction (all files) 2024-11-14T07:27:01,228 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/B in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,229 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e35d47d4628f459492dc56fc03862bf2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=48.6 K 2024-11-14T07:27:01,229 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54630 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:01,229 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/A is initiating minor compaction (all files) 2024-11-14T07:27:01,229 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/A in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,229 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/d9e4deddd0de4784bb20d71b00f73a7f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=53.3 K 2024-11-14T07:27:01,229 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e35d47d4628f459492dc56fc03862bf2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:27:01,230 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c17075f01944e09f8c11e4c3f9bdbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731569219032 2024-11-14T07:27:01,230 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e4deddd0de4784bb20d71b00f73a7f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:27:01,231 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca85c7fe3c745febdeb552790e2c103, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731569219032 2024-11-14T07:27:01,231 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting ed308af2e8274f47932c06e79f9db901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731569219089 2024-11-14T07:27:01,231 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1711da475c8496dade23820b5c53abc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731569219089 2024-11-14T07:27:01,231 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2442f4b63fdd46f2bd03caa44ba29244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731569220380 2024-11-14T07:27:01,231 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7052d2fc3fcf4a9aab85d895284841b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731569220380 2024-11-14T07:27:01,243 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:01,243 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ee5a9f9923c54ff498b1b98c90ac9837 is 50, key is test_row_0/B:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:01,246 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#A#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:01,246 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/18ef7e5c275244059a4a056a3d11420a is 50, key is test_row_0/A:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742400_1576 (size=12983) 2024-11-14T07:27:01,279 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/ee5a9f9923c54ff498b1b98c90ac9837 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ee5a9f9923c54ff498b1b98c90ac9837 2024-11-14T07:27:01,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742399_1575 (size=12983) 2024-11-14T07:27:01,283 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/B of 5f1c5a6a1ccc8a28937c7c671893d5fc into ee5a9f9923c54ff498b1b98c90ac9837(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:01,283 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:01,283 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/B, priority=12, startTime=1731569221226; duration=0sec 2024-11-14T07:27:01,283 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:01,283 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:B 2024-11-14T07:27:01,283 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:01,287 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:01,287 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 5f1c5a6a1ccc8a28937c7c671893d5fc/C is initiating minor compaction (all files) 2024-11-14T07:27:01,287 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f1c5a6a1ccc8a28937c7c671893d5fc/C in TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,287 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5d13374b6c3e4b96b632c4d7ccc7210d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp, totalSize=48.6 K 2024-11-14T07:27:01,288 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d13374b6c3e4b96b632c4d7ccc7210d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731569218287 2024-11-14T07:27:01,288 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 276aaed1d2cc468183e79d4247353d14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731569219032 2024-11-14T07:27:01,289 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c8f12f080f4d94b840c8e6147c9265, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731569219089 2024-11-14T07:27:01,289 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dd1f4626b964a4bba850b95aefcaf60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731569220380 2024-11-14T07:27:01,297 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f1c5a6a1ccc8a28937c7c671893d5fc#C#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:01,298 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/ec3a1dab6d2b4a33a6f198ce9210421b is 50, key is test_row_0/C:col10/1731569220728/Put/seqid=0 2024-11-14T07:27:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742401_1577 (size=12983) 2024-11-14T07:27:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:01,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:27:01,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/ef6d57f8a1064ddc899919e17f133805 is 50, key is test_row_0/A:col10/1731569221429/Put/seqid=0 2024-11-14T07:27:01,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742402_1578 (size=17181) 2024-11-14T07:27:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T07:27:01,545 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:01,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-14T07:27:01,552 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:01,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:01,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569281577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569281578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:01,686 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/18ef7e5c275244059a4a056a3d11420a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/18ef7e5c275244059a4a056a3d11420a 2024-11-14T07:27:01,692 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/A of 5f1c5a6a1ccc8a28937c7c671893d5fc into 18ef7e5c275244059a4a056a3d11420a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:01,692 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:01,692 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/A, priority=12, startTime=1731569221226; duration=0sec 2024-11-14T07:27:01,692 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:01,692 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:A 2024-11-14T07:27:01,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-14T07:27:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569281695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,708 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/ec3a1dab6d2b4a33a6f198ce9210421b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/ec3a1dab6d2b4a33a6f198ce9210421b 2024-11-14T07:27:01,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569281695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,716 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f1c5a6a1ccc8a28937c7c671893d5fc/C of 5f1c5a6a1ccc8a28937c7c671893d5fc into ec3a1dab6d2b4a33a6f198ce9210421b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:01,716 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:01,716 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., storeName=5f1c5a6a1ccc8a28937c7c671893d5fc/C, priority=12, startTime=1731569221228; duration=0sec 2024-11-14T07:27:01,716 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:01,716 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f1c5a6a1ccc8a28937c7c671893d5fc:C 2024-11-14T07:27:01,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/ef6d57f8a1064ddc899919e17f133805 2024-11-14T07:27:01,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/26b9f03edb8545fb8a4b0722e537b2cd is 50, key is test_row_0/B:col10/1731569221429/Put/seqid=0 2024-11-14T07:27:01,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742403_1579 (size=12301) 2024-11-14T07:27:01,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/26b9f03edb8545fb8a4b0722e537b2cd 2024-11-14T07:27:01,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-14T07:27:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:01,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:01,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/d590feec15164e56864a9e0c92a30ea8 is 50, key is test_row_0/C:col10/1731569221429/Put/seqid=0 2024-11-14T07:27:01,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742404_1580 (size=12301) 2024-11-14T07:27:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569281914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569281915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:02,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-14T07:27:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:02,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-14T07:27:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:02,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:02,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44616 deadline: 1731569282225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:02,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44596 deadline: 1731569282225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 , the old value is region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=2 because the exception is null or not the one we care about 2024-11-14T07:27:02,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/d590feec15164e56864a9e0c92a30ea8 2024-11-14T07:27:02,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/ef6d57f8a1064ddc899919e17f133805 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/ef6d57f8a1064ddc899919e17f133805 2024-11-14T07:27:02,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/ef6d57f8a1064ddc899919e17f133805, entries=250, sequenceid=376, filesize=16.8 K 2024-11-14T07:27:02,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/26b9f03edb8545fb8a4b0722e537b2cd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/26b9f03edb8545fb8a4b0722e537b2cd 2024-11-14T07:27:02,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/26b9f03edb8545fb8a4b0722e537b2cd, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T07:27:02,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/d590feec15164e56864a9e0c92a30ea8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d590feec15164e56864a9e0c92a30ea8 2024-11-14T07:27:02,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d590feec15164e56864a9e0c92a30ea8, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T07:27:02,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 853ms, sequenceid=376, compaction requested=false 2024-11-14T07:27:02,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:02,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-14T07:27:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:02,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:27:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f115526a16664d068ca7479736fcfa40 is 50, key is test_row_0/A:col10/1731569221545/Put/seqid=0 2024-11-14T07:27:02,325 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-11-14T07:27:02,325 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,325 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,325 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742405_1581 (size=12301) 2024-11-14T07:27:02,326 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-14T07:27:02,326 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,326 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,326 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,329 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-14T07:27:02,329 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,329 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,329 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,331 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-14T07:27:02,331 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,331 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,331 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,332 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-11-14T07:27:02,332 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,332 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,332 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:02,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f115526a16664d068ca7479736fcfa40 2024-11-14T07:27:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/87e4b19e61f7472bb9b0799c71394d3e is 50, key is test_row_0/B:col10/1731569221545/Put/seqid=0 2024-11-14T07:27:02,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742406_1582 (size=12301) 2024-11-14T07:27:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. as already flushing 2024-11-14T07:27:02,736 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-14T07:27:02,736 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-14T07:27:02,736 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,736 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:02,736 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,736 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,736 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,736 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:02,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:02,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:03,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/87e4b19e61f7472bb9b0799c71394d3e 2024-11-14T07:27:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6f2976e11af545658087b32eb105fc16 is 50, key is test_row_0/C:col10/1731569221545/Put/seqid=0 2024-11-14T07:27:03,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742407_1583 (size=12301) 2024-11-14T07:27:03,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6f2976e11af545658087b32eb105fc16 2024-11-14T07:27:03,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/f115526a16664d068ca7479736fcfa40 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f115526a16664d068ca7479736fcfa40 2024-11-14T07:27:03,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f115526a16664d068ca7479736fcfa40, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T07:27:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/87e4b19e61f7472bb9b0799c71394d3e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/87e4b19e61f7472bb9b0799c71394d3e 2024-11-14T07:27:03,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/87e4b19e61f7472bb9b0799c71394d3e, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T07:27:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/6f2976e11af545658087b32eb105fc16 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6f2976e11af545658087b32eb105fc16 2024-11-14T07:27:03,554 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6f2976e11af545658087b32eb105fc16, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T07:27:03,554 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=13.42 KB/13740 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1236ms, sequenceid=401, compaction requested=true 2024-11-14T07:27:03,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: 2024-11-14T07:27:03,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:03,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-14T07:27:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-14T07:27:03,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T07:27:03,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0020 sec 2024-11-14T07:27:03,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 2.0120 sec 2024-11-14T07:27:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T07:27:03,705 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:09,265 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-14T07:27:09,265 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:09,265 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,265 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:09,306 INFO [Thread-2201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2201. 2024-11-14T07:27:09,306 DEBUG [Thread-2201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:09,306 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,306 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:09,316 INFO [Thread-2203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2203. 2024-11-14T07:27:09,316 DEBUG [Thread-2203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:09,316 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,316 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1785 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5351 rows 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1775 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5325 rows 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1780 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5337 rows 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1779 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5329 rows 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1781 2024-11-14T07:27:09,317 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5342 rows 2024-11-14T07:27:09,317 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:27:09,317 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:27:09,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:09,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:09,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:27:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:09,320 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569229320"}]},"ts":"1731569229320"} 2024-11-14T07:27:09,321 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:27:09,321 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:27:09,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:27:09,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, UNASSIGN}] 2024-11-14T07:27:09,323 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, UNASSIGN 2024-11-14T07:27:09,323 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5f1c5a6a1ccc8a28937c7c671893d5fc, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:09,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, UNASSIGN because future has completed 2024-11-14T07:27:09,325 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:27:09,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:27:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:09,477 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:09,477 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:27:09,477 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 5f1c5a6a1ccc8a28937c7c671893d5fc, disabling compactions & flushes 2024-11-14T07:27:09,477 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:09,477 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:09,477 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. after waiting 0 ms 2024-11-14T07:27:09,477 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:09,477 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T07:27:09,481 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a859b3d9bd0543d0a41db2dc077a0a75 is 50, key is test_row_0/A:col10/1731569229305/Put/seqid=0 2024-11-14T07:27:09,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742408_1584 (size=12301) 2024-11-14T07:27:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:09,885 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a859b3d9bd0543d0a41db2dc077a0a75 2024-11-14T07:27:09,890 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/919109b1b4754268ac0a060ea2fcd8c9 is 50, key is test_row_0/B:col10/1731569229305/Put/seqid=0 2024-11-14T07:27:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742409_1585 (size=12301) 2024-11-14T07:27:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:10,294 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/919109b1b4754268ac0a060ea2fcd8c9 2024-11-14T07:27:10,299 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/708dcc25aea14625b6042a1079f74f9b is 50, key is test_row_0/C:col10/1731569229305/Put/seqid=0 2024-11-14T07:27:10,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742410_1586 (size=12301) 2024-11-14T07:27:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:10,703 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/708dcc25aea14625b6042a1079f74f9b 2024-11-14T07:27:10,706 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/A/a859b3d9bd0543d0a41db2dc077a0a75 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a859b3d9bd0543d0a41db2dc077a0a75 2024-11-14T07:27:10,709 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a859b3d9bd0543d0a41db2dc077a0a75, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T07:27:10,710 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/B/919109b1b4754268ac0a060ea2fcd8c9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/919109b1b4754268ac0a060ea2fcd8c9 2024-11-14T07:27:10,713 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/919109b1b4754268ac0a060ea2fcd8c9, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T07:27:10,714 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/.tmp/C/708dcc25aea14625b6042a1079f74f9b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/708dcc25aea14625b6042a1079f74f9b 2024-11-14T07:27:10,716 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/708dcc25aea14625b6042a1079f74f9b, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T07:27:10,717 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1240ms, sequenceid=409, compaction requested=true 2024-11-14T07:27:10,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e1b803aed06440578b96b9dde45cad9d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/db883680972a436a8ae677769a70bdcf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/fe720b4a718f463eb72467a27060625c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/2d81a04a258d4d758206d932e8dcf846, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/598aa62cb0f245a08bf9c5c4e880c46c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/d9e4deddd0de4784bb20d71b00f73a7f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5] to archive 2024-11-14T07:27:10,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:10,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e1b803aed06440578b96b9dde45cad9d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e1b803aed06440578b96b9dde45cad9d 2024-11-14T07:27:10,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/648cf744e3484a0684e789bbae9a6833 2024-11-14T07:27:10,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/db883680972a436a8ae677769a70bdcf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/db883680972a436a8ae677769a70bdcf 2024-11-14T07:27:10,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/8d43b6da42e54d30b44afca449414c7a 2024-11-14T07:27:10,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5c8ab1b758654d909370146c5d5f1fc0 2024-11-14T07:27:10,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/5487bc09cdca418292568dcaada01acd 2024-11-14T07:27:10,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/fe720b4a718f463eb72467a27060625c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/fe720b4a718f463eb72467a27060625c 2024-11-14T07:27:10,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e4b4513c91854126b12b9fe24d700dda 2024-11-14T07:27:10,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/222cfcd3d9be47a6852fe1fc329e0daf 2024-11-14T07:27:10,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/2d81a04a258d4d758206d932e8dcf846 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/2d81a04a258d4d758206d932e8dcf846 2024-11-14T07:27:10,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a1627ac908704d5faeeb2b1d38d26778 2024-11-14T07:27:10,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7a2e6aee80ac4a35ad40372e9dd7163b 2024-11-14T07:27:10,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/e82e642d7e2745a99976fb2185d6a113 2024-11-14T07:27:10,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/598aa62cb0f245a08bf9c5c4e880c46c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/598aa62cb0f245a08bf9c5c4e880c46c 2024-11-14T07:27:10,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/08d64f96b1d84a1891fc7d85205b9c09 2024-11-14T07:27:10,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c5d4e52fa0ab46be95324270217e3fe8 2024-11-14T07:27:10,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/d9e4deddd0de4784bb20d71b00f73a7f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/d9e4deddd0de4784bb20d71b00f73a7f 2024-11-14T07:27:10,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/9ca85c7fe3c745febdeb552790e2c103 2024-11-14T07:27:10,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/c1711da475c8496dade23820b5c53abc 2024-11-14T07:27:10,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/7052d2fc3fcf4a9aab85d895284841b5 2024-11-14T07:27:10,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/fc5b862eaae44746becf414d14742559, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/7c203f727c764c16b7114e87ce830f53, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2dba4e8b229d49a296cc2ab99685ee49, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/cb798d98359e4a10b985dc939463da55, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1b3ff684339a4758b6036438fc7330f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e35d47d4628f459492dc56fc03862bf2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244] to archive 2024-11-14T07:27:10,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:10,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/fc5b862eaae44746becf414d14742559 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/fc5b862eaae44746becf414d14742559 2024-11-14T07:27:10,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/d9fae247487d4608a7010929f61c248e 2024-11-14T07:27:10,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/7c203f727c764c16b7114e87ce830f53 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/7c203f727c764c16b7114e87ce830f53 2024-11-14T07:27:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/75c2ad1a8fe04e04baaa9e3bc7b7ff85 2024-11-14T07:27:10,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2bc338de23834b408f769c6fbe95d6ae 2024-11-14T07:27:10,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2dba4e8b229d49a296cc2ab99685ee49 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2dba4e8b229d49a296cc2ab99685ee49 2024-11-14T07:27:10,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/de58d0368bdb4d9791054825d28fdb71 2024-11-14T07:27:10,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/4a324bead5c54cf18d2592423845c519 2024-11-14T07:27:10,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/cb798d98359e4a10b985dc939463da55 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/cb798d98359e4a10b985dc939463da55 2024-11-14T07:27:10,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/23f153ab0b6946a9b8572b44a195c91e 2024-11-14T07:27:10,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/25f659123edf421983cb10bece4bdc7d 2024-11-14T07:27:10,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/9e290d174eff455cb92d9bbe0a768de4 2024-11-14T07:27:10,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1b3ff684339a4758b6036438fc7330f2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/1b3ff684339a4758b6036438fc7330f2 2024-11-14T07:27:10,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ebfe9b7250fd49c09940bf748b662640 2024-11-14T07:27:10,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/967961d533874dfca67ed30fed2113c7 2024-11-14T07:27:10,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e35d47d4628f459492dc56fc03862bf2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e35d47d4628f459492dc56fc03862bf2 2024-11-14T07:27:10,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/e9129f4f491c4252a4834bddf29946b8 2024-11-14T07:27:10,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/00c17075f01944e09f8c11e4c3f9bdbe 2024-11-14T07:27:10,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ed308af2e8274f47932c06e79f9db901 2024-11-14T07:27:10,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/2442f4b63fdd46f2bd03caa44ba29244 2024-11-14T07:27:10,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d15a465385e7430aa78069505e1bc0a8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/1f3e8b9b128f4767bef46528a0b28f33, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a412a1b001a5403582b4cb01910ba484, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a2700b8b58314d98925887905b520d37, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/747540e6428942cd82d02d64d3826c91, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5d13374b6c3e4b96b632c4d7ccc7210d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60] to archive 2024-11-14T07:27:10,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:10,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d15a465385e7430aa78069505e1bc0a8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d15a465385e7430aa78069505e1bc0a8 2024-11-14T07:27:10,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/3040d309bfa04e46970d27176c684a69 2024-11-14T07:27:10,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/1f3e8b9b128f4767bef46528a0b28f33 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/1f3e8b9b128f4767bef46528a0b28f33 2024-11-14T07:27:10,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/28999bab919341d59e04858fc59aac1c 2024-11-14T07:27:10,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bc8d6e181b914dce807e1da2e1a245f7 2024-11-14T07:27:10,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a412a1b001a5403582b4cb01910ba484 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a412a1b001a5403582b4cb01910ba484 2024-11-14T07:27:10,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/c1bfdfcd895e40c2a9a357ca55aa826f 2024-11-14T07:27:10,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5f3421c295c548d1a18a3a7f29417cc8 2024-11-14T07:27:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a2700b8b58314d98925887905b520d37 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/a2700b8b58314d98925887905b520d37 2024-11-14T07:27:10,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/bf0927e1fb1b49ee9a0860da640c1d14 2024-11-14T07:27:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/12a24055715f4d7b9cbdd3f520537d0c 2024-11-14T07:27:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6b75e30773ea4abdb011e4c3b165ab19 2024-11-14T07:27:10,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/747540e6428942cd82d02d64d3826c91 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/747540e6428942cd82d02d64d3826c91 2024-11-14T07:27:10,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/152d2c987bd14a77b6dddd75c81d8f65 2024-11-14T07:27:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/eeace6d854104404acf62fa41d939e25 2024-11-14T07:27:10,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5d13374b6c3e4b96b632c4d7ccc7210d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5d13374b6c3e4b96b632c4d7ccc7210d 2024-11-14T07:27:10,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/937da354550b4dda8e0fc942ef811f58 2024-11-14T07:27:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/276aaed1d2cc468183e79d4247353d14 2024-11-14T07:27:10,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/88c8f12f080f4d94b840c8e6147c9265 2024-11-14T07:27:10,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/5dd1f4626b964a4bba850b95aefcaf60 2024-11-14T07:27:10,788 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=1 2024-11-14T07:27:10,788 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. 2024-11-14T07:27:10,789 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 5f1c5a6a1ccc8a28937c7c671893d5fc: Waiting for close lock at 1731569229477Running coprocessor pre-close hooks at 1731569229477Disabling compacts and flushes for region at 1731569229477Disabling writes for close at 1731569229477Obtaining lock to block concurrent updates at 1731569229477Preparing flush snapshotting stores in 5f1c5a6a1ccc8a28937c7c671893d5fc at 1731569229477Finished memstore snapshotting TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731569229478 (+1 ms)Flushing stores of TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc. at 1731569229478Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/A: creating writer at 1731569229478Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/A: appending metadata at 1731569229481 (+3 ms)Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/A: closing flushed file at 1731569229481Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/B: creating writer at 1731569229888 (+407 ms)Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/B: appending metadata at 1731569229890 (+2 ms)Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/B: closing flushed file at 1731569229890Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/C: creating writer at 1731569230297 (+407 ms)Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/C: appending metadata at 1731569230299 (+2 ms)Flushing 5f1c5a6a1ccc8a28937c7c671893d5fc/C: closing flushed file at 1731569230299Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60a245a2: reopening flushed file at 1731569230706 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4122830: reopening flushed file at 1731569230710 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d20fa07: reopening flushed file at 1731569230713 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5f1c5a6a1ccc8a28937c7c671893d5fc in 1240ms, sequenceid=409, compaction requested=true at 1731569230717 (+4 ms)Writing region close event to WAL at 1731569230780 (+63 ms)Running coprocessor post-close hooks at 1731569230788 (+8 ms)Closed at 1731569230788 2024-11-14T07:27:10,791 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:10,791 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5f1c5a6a1ccc8a28937c7c671893d5fc, regionState=CLOSED 2024-11-14T07:27:10,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:27:10,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-14T07:27:10,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f1c5a6a1ccc8a28937c7c671893d5fc, server=c7aa6e2790b3,46235,1731569085523 in 1.4680 sec 2024-11-14T07:27:10,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T07:27:10,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f1c5a6a1ccc8a28937c7c671893d5fc, UNASSIGN in 1.4730 sec 2024-11-14T07:27:10,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T07:27:10,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4760 sec 2024-11-14T07:27:10,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569230800"}]},"ts":"1731569230800"} 2024-11-14T07:27:10,801 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:27:10,801 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:27:10,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-14T07:27:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T07:27:11,465 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:11,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:27:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,467 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-14T07:27:11,467 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,470 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:11,472 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/recovered.edits] 2024-11-14T07:27:11,474 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/18ef7e5c275244059a4a056a3d11420a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/18ef7e5c275244059a4a056a3d11420a 2024-11-14T07:27:11,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a859b3d9bd0543d0a41db2dc077a0a75 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/a859b3d9bd0543d0a41db2dc077a0a75 2024-11-14T07:27:11,476 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/ef6d57f8a1064ddc899919e17f133805 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/ef6d57f8a1064ddc899919e17f133805 2024-11-14T07:27:11,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f115526a16664d068ca7479736fcfa40 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/A/f115526a16664d068ca7479736fcfa40 2024-11-14T07:27:11,478 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/26b9f03edb8545fb8a4b0722e537b2cd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/26b9f03edb8545fb8a4b0722e537b2cd 2024-11-14T07:27:11,479 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/87e4b19e61f7472bb9b0799c71394d3e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/87e4b19e61f7472bb9b0799c71394d3e 2024-11-14T07:27:11,479 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/919109b1b4754268ac0a060ea2fcd8c9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/919109b1b4754268ac0a060ea2fcd8c9 2024-11-14T07:27:11,480 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ee5a9f9923c54ff498b1b98c90ac9837 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/B/ee5a9f9923c54ff498b1b98c90ac9837 2024-11-14T07:27:11,482 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6f2976e11af545658087b32eb105fc16 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/6f2976e11af545658087b32eb105fc16 2024-11-14T07:27:11,483 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/708dcc25aea14625b6042a1079f74f9b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/708dcc25aea14625b6042a1079f74f9b 2024-11-14T07:27:11,484 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d590feec15164e56864a9e0c92a30ea8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/d590feec15164e56864a9e0c92a30ea8 2024-11-14T07:27:11,484 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/ec3a1dab6d2b4a33a6f198ce9210421b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/C/ec3a1dab6d2b4a33a6f198ce9210421b 2024-11-14T07:27:11,486 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/recovered.edits/412.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc/recovered.edits/412.seqid 2024-11-14T07:27:11,487 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/5f1c5a6a1ccc8a28937c7c671893d5fc 2024-11-14T07:27:11,487 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:27:11,489 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,490 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:27:11,492 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:27:11,493 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,493 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:27:11,494 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569231493"}]},"ts":"9223372036854775807"} 2024-11-14T07:27:11,496 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:27:11,496 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5f1c5a6a1ccc8a28937c7c671893d5fc, NAME => 'TestAcidGuarantees,,1731569201130.5f1c5a6a1ccc8a28937c7c671893d5fc.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:27:11,496 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:27:11,496 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569231496"}]},"ts":"9223372036854775807"} 2024-11-14T07:27:11,497 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:27:11,506 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-14T07:27:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-14T07:27:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:27:11,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:11,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 510), ProcessCount=11 (was 11), AvailableMemoryMB=5465 (was 5598) 2024-11-14T07:27:11,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=5465 2024-11-14T07:27:11,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:27:11,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:27:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:11,597 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T07:27:11,597 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:11,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-14T07:27:11,598 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T07:27:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:11,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742411_1587 (size=959) 2024-11-14T07:27:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:12,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59 2024-11-14T07:27:12,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742412_1588 (size=53) 2024-11-14T07:27:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:12,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:27:12,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4687db304ff704d35e9eef31fb5612b9, disabling compactions & flushes 2024-11-14T07:27:12,410 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,410 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,410 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. after waiting 0 ms 2024-11-14T07:27:12,410 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,410 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,410 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4687db304ff704d35e9eef31fb5612b9: Waiting for close lock at 1731569232409Disabling compacts and flushes for region at 1731569232409Disabling writes for close at 1731569232410 (+1 ms)Writing region close event to WAL at 1731569232410Closed at 1731569232410 2024-11-14T07:27:12,411 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T07:27:12,411 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731569232411"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731569232411"}]},"ts":"1731569232411"} 2024-11-14T07:27:12,412 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T07:27:12,413 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T07:27:12,413 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569232413"}]},"ts":"1731569232413"} 2024-11-14T07:27:12,414 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T07:27:12,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, ASSIGN}] 2024-11-14T07:27:12,415 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, ASSIGN 2024-11-14T07:27:12,416 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, ASSIGN; state=OFFLINE, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=false 2024-11-14T07:27:12,566 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:12,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, ASSIGN because future has completed 2024-11-14T07:27:12,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:27:12,723 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,723 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:27:12,723 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,723 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:27:12,723 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,723 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,725 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,726 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName A 2024-11-14T07:27:12,726 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:12,726 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:12,727 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,728 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName B 2024-11-14T07:27:12,728 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:12,728 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:12,728 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,729 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName C 2024-11-14T07:27:12,729 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:12,730 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:12,730 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,730 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,731 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,732 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,732 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,732 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:27:12,733 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:12,735 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T07:27:12,736 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 4687db304ff704d35e9eef31fb5612b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64666459, jitterRate=-0.036394670605659485}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:27:12,736 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:12,736 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 4687db304ff704d35e9eef31fb5612b9: Running coprocessor pre-open hook at 1731569232724Writing region info on filesystem at 1731569232724Initializing all the Stores at 1731569232724Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569232724Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569232724Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569232724Cleaning up temporary data from old regions at 1731569232732 (+8 ms)Running coprocessor post-open hooks at 1731569232736 (+4 ms)Region opened successfully at 1731569232736 2024-11-14T07:27:12,737 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., pid=152, masterSystemTime=1731569232720 2024-11-14T07:27:12,739 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,739 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:12,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=OPEN, openSeqNum=2, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:12,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:27:12,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-14T07:27:12,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 in 173 msec 2024-11-14T07:27:12,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T07:27:12,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, ASSIGN in 329 msec 2024-11-14T07:27:12,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T07:27:12,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569232745"}]},"ts":"1731569232745"} 2024-11-14T07:27:12,746 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T07:27:12,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T07:27:12,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-14T07:27:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T07:27:13,745 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:13,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:13,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:13,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:13,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9b28aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:13,749 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54bca363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:13,750 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:13,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:13,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T07:27:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c7aa6e2790b3,42927,1731569084750 2024-11-14T07:27:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67e7c0ae 2024-11-14T07:27:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T07:27:13,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T07:27:13,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T07:27:13,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T07:27:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:13,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742413_1589 (size=995) 2024-11-14T07:27:13,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:27:14,166 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T07:27:14,166 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T07:27:14,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:27:14,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE}] 2024-11-14T07:27:14,170 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE 2024-11-14T07:27:14,170 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE because future has completed 2024-11-14T07:27:14,171 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:27:14,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:27:14,323 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,323 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:27:14,324 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 4687db304ff704d35e9eef31fb5612b9, disabling compactions & flushes 2024-11-14T07:27:14,324 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,324 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,324 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. after waiting 0 ms 2024-11-14T07:27:14,324 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,326 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T07:27:14,327 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,327 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 4687db304ff704d35e9eef31fb5612b9: Waiting for close lock at 1731569234323Running coprocessor pre-close hooks at 1731569234323Disabling compacts and flushes for region at 1731569234323Disabling writes for close at 1731569234324 (+1 ms)Writing region close event to WAL at 1731569234324Running coprocessor post-close hooks at 1731569234327 (+3 ms)Closed at 1731569234327 2024-11-14T07:27:14,327 WARN [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 4687db304ff704d35e9eef31fb5612b9 to self. 2024-11-14T07:27:14,329 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,329 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=CLOSED 2024-11-14T07:27:14,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:27:14,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-14T07:27:14,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 in 160 msec 2024-11-14T07:27:14,338 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE; state=CLOSED, location=c7aa6e2790b3,46235,1731569085523; forceNewPlan=false, retain=true 2024-11-14T07:27:14,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=OPENING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE because future has completed 2024-11-14T07:27:14,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:27:14,644 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,644 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} 2024-11-14T07:27:14,645 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,645 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T07:27:14,645 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,645 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,646 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,646 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName A 2024-11-14T07:27:14,647 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:14,647 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:14,648 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,648 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName B 2024-11-14T07:27:14,648 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:14,648 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:14,648 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,649 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4687db304ff704d35e9eef31fb5612b9 columnFamilyName C 2024-11-14T07:27:14,649 DEBUG [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:14,649 INFO [StoreOpener-4687db304ff704d35e9eef31fb5612b9-1 {}] regionserver.HStore(327): Store=4687db304ff704d35e9eef31fb5612b9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T07:27:14,649 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,649 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,650 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,651 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,651 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,651 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T07:27:14,652 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,652 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 4687db304ff704d35e9eef31fb5612b9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69533569, jitterRate=0.03613092005252838}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T07:27:14,652 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,653 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 4687db304ff704d35e9eef31fb5612b9: Running coprocessor pre-open hook at 1731569234645Writing region info on filesystem at 1731569234645Initializing all the Stores at 1731569234645Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569234645Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569234645Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731569234645Cleaning up temporary data from old regions at 1731569234651 (+6 ms)Running coprocessor post-open hooks at 1731569234652 (+1 ms)Region opened successfully at 1731569234653 (+1 ms) 2024-11-14T07:27:14,653 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., pid=157, masterSystemTime=1731569234642 2024-11-14T07:27:14,655 DEBUG [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,655 INFO [RS_OPEN_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=OPEN, openSeqNum=5, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:27:14,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-14T07:27:14,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 in 167 msec 2024-11-14T07:27:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T07:27:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, REOPEN/MOVE in 490 msec 2024-11-14T07:27:14,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-14T07:27:14,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-14T07:27:14,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-14T07:27:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-14T07:27:14,665 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:14,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4325cfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab0502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6b721b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4870f2e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,672 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115ec7bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ed7e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15bded0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,679 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3ecef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd97a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1309d422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,683 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141cf8d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,685 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18e65132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c648753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e6283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a1624b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df403b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7e240f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c7aa6e2790b3,42927,-1 for getting cluster id 2024-11-14T07:27:14,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T07:27:14,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b6ed1909-abed-4236-8083-e38d7f3d917f' 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b6ed1909-abed-4236-8083-e38d7f3d917f" 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f45326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c7aa6e2790b3,42927,-1] 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T07:27:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:14,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T07:27:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a725f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T07:27:14,697 DEBUG [Thread-2612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,697 DEBUG [Thread-2609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,698 DEBUG [Thread-2618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:14,698 DEBUG [Thread-2601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,699 DEBUG [Thread-2603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T07:27:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,700 DEBUG [Thread-2620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,700 DEBUG [Thread-2614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,701 DEBUG [Thread-2607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,702 DEBUG [Thread-2605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:14,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,704 DEBUG [Thread-2616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T07:27:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c7aa6e2790b3,46235,1731569085523, seqNum=-1] 2024-11-14T07:27:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T07:27:14,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T07:27:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5] 2024-11-14T07:27:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:14,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:14,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:27:14,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148a770e08957343d695dbc5e8893de847_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569234705/Put/seqid=0 2024-11-14T07:27:14,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742414_1590 (size=14594) 2024-11-14T07:27:14,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569294768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569294770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569294773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569294773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569294773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:14,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:14,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:14,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:14,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569295095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569295095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569295095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569295095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569295095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:15,143 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148a770e08957343d695dbc5e8893de847_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a770e08957343d695dbc5e8893de847_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:15,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/3c3287f7aa604065b684862a6e29c057, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:15,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/3c3287f7aa604065b684862a6e29c057 is 175, key is test_row_0/A:col10/1731569234705/Put/seqid=0 2024-11-14T07:27:15,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742415_1591 (size=39549) 2024-11-14T07:27:15,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569295404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569295404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569295404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569295404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569295407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/3c3287f7aa604065b684862a6e29c057 2024-11-14T07:27:15,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/89074f46cf174c33a4e38f8619d1798e is 50, key is test_row_0/B:col10/1731569234705/Put/seqid=0 2024-11-14T07:27:15,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742416_1592 (size=12001) 2024-11-14T07:27:15,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569295915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569295915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569295915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569295915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569295915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:15,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:15,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:16,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/89074f46cf174c33a4e38f8619d1798e 2024-11-14T07:27:16,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/d9ff4de211004ffc85125bdae8e0ec65 is 50, key is test_row_0/C:col10/1731569234705/Put/seqid=0 2024-11-14T07:27:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742417_1593 (size=12001) 2024-11-14T07:27:16,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/d9ff4de211004ffc85125bdae8e0ec65 2024-11-14T07:27:16,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/3c3287f7aa604065b684862a6e29c057 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057 2024-11-14T07:27:16,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057, entries=200, sequenceid=15, filesize=38.6 K 2024-11-14T07:27:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/89074f46cf174c33a4e38f8619d1798e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e 2024-11-14T07:27:16,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:27:16,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/d9ff4de211004ffc85125bdae8e0ec65 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65 2024-11-14T07:27:16,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T07:27:16,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 4687db304ff704d35e9eef31fb5612b9 in 1369ms, sequenceid=15, compaction requested=false 2024-11-14T07:27:16,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:16,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T07:27:16,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:16,091 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T07:27:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee4c67e41c4648d2bda7ca7673593997_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569234771/Put/seqid=0 2024-11-14T07:27:16,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742418_1594 (size=12154) 2024-11-14T07:27:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee4c67e41c4648d2bda7ca7673593997_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee4c67e41c4648d2bda7ca7673593997_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:16,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/1a98a61907d541f3b468ff42cf20072a, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:16,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/1a98a61907d541f3b468ff42cf20072a is 175, key is test_row_0/A:col10/1731569234771/Put/seqid=0 2024-11-14T07:27:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742419_1595 (size=30955) 2024-11-14T07:27:16,145 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/1a98a61907d541f3b468ff42cf20072a 2024-11-14T07:27:16,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/7ccd46f6cdc348e18d3f5002559238b2 is 50, key is test_row_0/B:col10/1731569234771/Put/seqid=0 2024-11-14T07:27:16,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742420_1596 (size=12001) 2024-11-14T07:27:16,305 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T07:27:16,586 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/7ccd46f6cdc348e18d3f5002559238b2 2024-11-14T07:27:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/04d24aac07004ff4abd96dc1ee5b29c0 is 50, key is test_row_0/C:col10/1731569234771/Put/seqid=0 2024-11-14T07:27:16,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742421_1597 (size=12001) 2024-11-14T07:27:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:16,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:16,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:16,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569296932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569296932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:16,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569296933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:16,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:16,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:16,936 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:16,936 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/04d24aac07004ff4abd96dc1ee5b29c0 2024-11-14T07:27:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/1a98a61907d541f3b468ff42cf20072a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a 2024-11-14T07:27:17,028 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a, entries=150, sequenceid=42, filesize=30.2 K 2024-11-14T07:27:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/7ccd46f6cdc348e18d3f5002559238b2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2 2024-11-14T07:27:17,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2, entries=150, sequenceid=42, filesize=11.7 K 2024-11-14T07:27:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/04d24aac07004ff4abd96dc1ee5b29c0 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0 2024-11-14T07:27:17,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0, entries=150, sequenceid=42, filesize=11.7 K 2024-11-14T07:27:17,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 4687db304ff704d35e9eef31fb5612b9 in 952ms, sequenceid=42, compaction requested=false 2024-11-14T07:27:17,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T07:27:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T07:27:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:17,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:27:17,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T07:27:17,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3420 sec 2024-11-14T07:27:17,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.3490 sec 2024-11-14T07:27:17,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111484155ebd707c4e848cd87073d2acb64c_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742422_1598 (size=14594) 2024-11-14T07:27:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569297104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569297104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569297104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569297215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569297215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569297215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569297425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569297425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569297425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:17,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111484155ebd707c4e848cd87073d2acb64c_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484155ebd707c4e848cd87073d2acb64c_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:17,464 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/daa9345dfe9c46e193aca994387ba1dc, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/daa9345dfe9c46e193aca994387ba1dc is 175, key is test_row_0/A:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742423_1599 (size=39549) 2024-11-14T07:27:17,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/daa9345dfe9c46e193aca994387ba1dc 2024-11-14T07:27:17,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/619bfa93465f47b2af68140dd28708ec is 50, key is test_row_0/B:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742424_1600 (size=12001) 2024-11-14T07:27:17,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/619bfa93465f47b2af68140dd28708ec 2024-11-14T07:27:17,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/71bb99e640a5433dbcb396c77c935264 is 50, key is test_row_0/C:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742425_1601 (size=12001) 2024-11-14T07:27:17,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/71bb99e640a5433dbcb396c77c935264 2024-11-14T07:27:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/daa9345dfe9c46e193aca994387ba1dc as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc 2024-11-14T07:27:17,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc, entries=200, sequenceid=54, filesize=38.6 K 2024-11-14T07:27:17,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/619bfa93465f47b2af68140dd28708ec as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec 2024-11-14T07:27:17,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T07:27:17,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/71bb99e640a5433dbcb396c77c935264 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264 2024-11-14T07:27:17,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T07:27:17,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4687db304ff704d35e9eef31fb5612b9 in 491ms, sequenceid=54, compaction requested=true 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:17,536 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:17,536 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:17,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:17,538 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:17,538 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:17,538 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=35.2 K 2024-11-14T07:27:17,538 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=107.5 K 2024-11-14T07:27:17,538 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc] 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 89074f46cf174c33a4e38f8619d1798e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569234705 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3287f7aa604065b684862a6e29c057, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569234705 2024-11-14T07:27:17,538 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ccd46f6cdc348e18d3f5002559238b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731569234765 2024-11-14T07:27:17,539 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 619bfa93465f47b2af68140dd28708ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:17,539 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a98a61907d541f3b468ff42cf20072a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731569234765 2024-11-14T07:27:17,539 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa9345dfe9c46e193aca994387ba1dc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:17,544 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,550 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:17,551 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/b066a02039744d12846fe623b8f0d2f2 is 50, key is test_row_0/B:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,552 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114316d7ab8a49c403190d6a2b546a16df9_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,554 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114316d7ab8a49c403190d6a2b546a16df9_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,554 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114316d7ab8a49c403190d6a2b546a16df9_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742427_1603 (size=4469) 2024-11-14T07:27:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742426_1602 (size=12104) 2024-11-14T07:27:17,572 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/b066a02039744d12846fe623b8f0d2f2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/b066a02039744d12846fe623b8f0d2f2 2024-11-14T07:27:17,576 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into b066a02039744d12846fe623b8f0d2f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:17,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:17,576 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569237536; duration=0sec 2024-11-14T07:27:17,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:17,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:17,576 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:17,577 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:17,577 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:17,577 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:17,577 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=35.2 K 2024-11-14T07:27:17,577 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d9ff4de211004ffc85125bdae8e0ec65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731569234705 2024-11-14T07:27:17,578 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d24aac07004ff4abd96dc1ee5b29c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731569234765 2024-11-14T07:27:17,578 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 71bb99e640a5433dbcb396c77c935264, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:17,583 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:17,583 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/aacdaacf3e454aef91df891039e7b491 is 50, key is test_row_0/C:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742428_1604 (size=12104) 2024-11-14T07:27:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:17,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:27:17,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa5586ebcb264350a0bd3b9a094ee91c_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569237737/Put/seqid=0 2024-11-14T07:27:17,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742429_1605 (size=12154) 2024-11-14T07:27:17,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:17,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569297747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569297748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa5586ebcb264350a0bd3b9a094ee91c_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa5586ebcb264350a0bd3b9a094ee91c_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:17,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9fb14bb87b75483eab44a956c8aec304, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:17,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9fb14bb87b75483eab44a956c8aec304 is 175, key is test_row_0/A:col10/1731569237737/Put/seqid=0 2024-11-14T07:27:17,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569297749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742430_1606 (size=30955) 2024-11-14T07:27:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569297854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569297854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569297864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:17,960 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#515 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:17,961 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f80060f4450f43ba9fd886bc81b11210 is 175, key is test_row_0/A:col10/1731569237045/Put/seqid=0 2024-11-14T07:27:17,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742431_1607 (size=31058) 2024-11-14T07:27:17,969 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f80060f4450f43ba9fd886bc81b11210 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210 2024-11-14T07:27:17,978 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into f80060f4450f43ba9fd886bc81b11210(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:17,978 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:17,979 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569237536; duration=0sec 2024-11-14T07:27:17,979 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:17,979 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:17,999 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/aacdaacf3e454aef91df891039e7b491 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/aacdaacf3e454aef91df891039e7b491 2024-11-14T07:27:18,003 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into aacdaacf3e454aef91df891039e7b491(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:18,003 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:18,003 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569237536; duration=0sec 2024-11-14T07:27:18,003 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:18,003 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569298064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569298064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569298074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9fb14bb87b75483eab44a956c8aec304 2024-11-14T07:27:18,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f043d444fc8d43d2b9cf722a3014766d is 50, key is test_row_0/B:col10/1731569237737/Put/seqid=0 2024-11-14T07:27:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742432_1608 (size=12001) 2024-11-14T07:27:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569298375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569298375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569298384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f043d444fc8d43d2b9cf722a3014766d 2024-11-14T07:27:18,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/6a6c02f5654f43e385084021d610b868 is 50, key is test_row_0/C:col10/1731569237737/Put/seqid=0 2024-11-14T07:27:18,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742433_1609 (size=12001) 2024-11-14T07:27:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T07:27:18,875 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:18,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T07:27:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:18,877 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:18,878 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:18,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:18,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569298884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569298884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569298894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569298955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569298955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:18,957 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,957 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:19,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/6a6c02f5654f43e385084021d610b868 2024-11-14T07:27:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9fb14bb87b75483eab44a956c8aec304 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304 2024-11-14T07:27:19,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304, entries=150, sequenceid=80, filesize=30.2 K 2024-11-14T07:27:19,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f043d444fc8d43d2b9cf722a3014766d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d 2024-11-14T07:27:19,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T07:27:19,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/6a6c02f5654f43e385084021d610b868 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868 2024-11-14T07:27:19,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868, entries=150, sequenceid=80, filesize=11.7 K 2024-11-14T07:27:19,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4687db304ff704d35e9eef31fb5612b9 in 1289ms, sequenceid=80, compaction requested=false 2024-11-14T07:27:19,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:19,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T07:27:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:19,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:27:19,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143c855211d2fe4a78b662eddbf21e91a1_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569237741/Put/seqid=0 2024-11-14T07:27:19,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742434_1610 (size=12154) 2024-11-14T07:27:19,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:19,078 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143c855211d2fe4a78b662eddbf21e91a1_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c855211d2fe4a78b662eddbf21e91a1_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/0c7b05a1acd94082823142467e692113, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/0c7b05a1acd94082823142467e692113 is 175, key is test_row_0/A:col10/1731569237741/Put/seqid=0 2024-11-14T07:27:19,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742435_1611 (size=30955) 2024-11-14T07:27:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:19,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/0c7b05a1acd94082823142467e692113 2024-11-14T07:27:19,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/8463d6de4cda49158588479cbb97a54d is 50, key is test_row_0/B:col10/1731569237741/Put/seqid=0 2024-11-14T07:27:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742436_1612 (size=12001) 2024-11-14T07:27:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:19,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/8463d6de4cda49158588479cbb97a54d 2024-11-14T07:27:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/75a75f30b6eb47adbb98fb93cf57313f is 50, key is test_row_0/C:col10/1731569237741/Put/seqid=0 2024-11-14T07:27:19,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742437_1613 (size=12001) 2024-11-14T07:27:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569299933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569299934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:19,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569299937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569300044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569300044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569300044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569300254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569300254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569300254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,330 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/75a75f30b6eb47adbb98fb93cf57313f 2024-11-14T07:27:20,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/0c7b05a1acd94082823142467e692113 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113 2024-11-14T07:27:20,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113, entries=150, sequenceid=93, filesize=30.2 K 2024-11-14T07:27:20,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/8463d6de4cda49158588479cbb97a54d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d 2024-11-14T07:27:20,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T07:27:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/75a75f30b6eb47adbb98fb93cf57313f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f 2024-11-14T07:27:20,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T07:27:20,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4687db304ff704d35e9eef31fb5612b9 in 1320ms, sequenceid=93, compaction requested=true 2024-11-14T07:27:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T07:27:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T07:27:20,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T07:27:20,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-14T07:27:20,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4790 sec 2024-11-14T07:27:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:20,567 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:20,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T07:27:20,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412b3ea5a55024f2dabdc17742da8ca3f_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:20,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742438_1614 (size=12154) 2024-11-14T07:27:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569300606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569300606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569300606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569300714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569300714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569300714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569300925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569300925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569300925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:20,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:20,982 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:20,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412b3ea5a55024f2dabdc17742da8ca3f_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412b3ea5a55024f2dabdc17742da8ca3f_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:20,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f8daf1dba5ca4f6c8dc5813c7084ceef, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:20,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f8daf1dba5ca4f6c8dc5813c7084ceef is 175, key is test_row_0/A:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:20,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742439_1615 (size=30955) 2024-11-14T07:27:20,997 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f8daf1dba5ca4f6c8dc5813c7084ceef 2024-11-14T07:27:21,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dde3caf788b481fa567700bc78f7219 is 50, key is test_row_0/B:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:21,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742440_1616 (size=12001) 2024-11-14T07:27:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T07:27:21,025 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:21,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T07:27:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:21,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:21,028 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:21,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:21,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569301234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569301235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569301235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:21,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dde3caf788b481fa567700bc78f7219 2024-11-14T07:27:21,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/84b06297336d4b1d8901104bf81af6d7 is 50, key is test_row_0/C:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:21,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742441_1617 (size=12001) 2024-11-14T07:27:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:21,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569301745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569301745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569301745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:21,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:21,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/84b06297336d4b1d8901104bf81af6d7 2024-11-14T07:27:21,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f8daf1dba5ca4f6c8dc5813c7084ceef as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef 2024-11-14T07:27:21,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef, entries=150, sequenceid=120, filesize=30.2 K 2024-11-14T07:27:21,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dde3caf788b481fa567700bc78f7219 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219 2024-11-14T07:27:21,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219, entries=150, sequenceid=120, filesize=11.7 K 2024-11-14T07:27:21,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/84b06297336d4b1d8901104bf81af6d7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7 2024-11-14T07:27:21,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7, entries=150, sequenceid=120, filesize=11.7 K 2024-11-14T07:27:21,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 4687db304ff704d35e9eef31fb5612b9 in 1266ms, sequenceid=120, compaction requested=true 2024-11-14T07:27:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:21,833 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:21,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:21,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:21,834 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:21,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:21,834 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:21,834 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:21,834 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:21,834 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:21,834 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,834 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,834 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/b066a02039744d12846fe623b8f0d2f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=47.0 K 2024-11-14T07:27:21,834 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=121.0 K 2024-11-14T07:27:21,835 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef] 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b066a02039744d12846fe623b8f0d2f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f043d444fc8d43d2b9cf722a3014766d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731569237075 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80060f4450f43ba9fd886bc81b11210, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8463d6de4cda49158588479cbb97a54d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731569237741 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb14bb87b75483eab44a956c8aec304, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731569237075 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dde3caf788b481fa567700bc78f7219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:21,835 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7b05a1acd94082823142467e692113, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731569237741 2024-11-14T07:27:21,836 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8daf1dba5ca4f6c8dc5813c7084ceef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:21,841 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:21,841 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:21,842 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/9bff1737522943f3a8e0b46b6bef600a is 50, key is test_row_0/B:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:21,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145d10e05b73c84499a8f6fb6d99622235_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:21,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742442_1618 (size=12241) 2024-11-14T07:27:21,851 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145d10e05b73c84499a8f6fb6d99622235_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:21,851 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145d10e05b73c84499a8f6fb6d99622235_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:21,854 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/9bff1737522943f3a8e0b46b6bef600a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/9bff1737522943f3a8e0b46b6bef600a 2024-11-14T07:27:21,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742443_1619 (size=4469) 2024-11-14T07:27:21,860 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into 9bff1737522943f3a8e0b46b6bef600a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:21,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:21,860 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=12, startTime=1731569241833; duration=0sec 2024-11-14T07:27:21,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:21,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:21,860 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:21,861 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:21,861 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:21,861 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,861 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/aacdaacf3e454aef91df891039e7b491, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=47.0 K 2024-11-14T07:27:21,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting aacdaacf3e454aef91df891039e7b491, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731569236926 2024-11-14T07:27:21,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6c02f5654f43e385084021d610b868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731569237075 2024-11-14T07:27:21,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a75f30b6eb47adbb98fb93cf57313f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731569237741 2024-11-14T07:27:21,862 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b06297336d4b1d8901104bf81af6d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:21,876 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#529 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:21,876 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/c1fe15ef40484af3ae0ecb57d4a174d7 is 50, key is test_row_0/C:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:21,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742444_1620 (size=12241) 2024-11-14T07:27:21,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T07:27:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T07:27:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142230ae5defc541328c30991b40ddcd39_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569240577/Put/seqid=0 2024-11-14T07:27:21,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742445_1621 (size=12204) 2024-11-14T07:27:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:21,958 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142230ae5defc541328c30991b40ddcd39_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142230ae5defc541328c30991b40ddcd39_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f13634efee6b4cdfbaf96c468ff17324, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f13634efee6b4cdfbaf96c468ff17324 is 175, key is test_row_0/A:col10/1731569240577/Put/seqid=0 2024-11-14T07:27:21,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742446_1622 (size=31005) 2024-11-14T07:27:21,974 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f13634efee6b4cdfbaf96c468ff17324 2024-11-14T07:27:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/5dabe642dc004484b08109dc2b644d39 is 50, key is test_row_0/B:col10/1731569240577/Put/seqid=0 2024-11-14T07:27:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742447_1623 (size=12051) 2024-11-14T07:27:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:22,257 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:22,258 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/d7225aa61ee94b49bf18b10613e57678 is 175, key is test_row_0/A:col10/1731569240566/Put/seqid=0 2024-11-14T07:27:22,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742448_1624 (size=31195) 2024-11-14T07:27:22,280 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/d7225aa61ee94b49bf18b10613e57678 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678 2024-11-14T07:27:22,284 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into d7225aa61ee94b49bf18b10613e57678(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:22,284 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:22,284 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=12, startTime=1731569241833; duration=0sec 2024-11-14T07:27:22,284 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:22,284 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:22,309 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/c1fe15ef40484af3ae0ecb57d4a174d7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/c1fe15ef40484af3ae0ecb57d4a174d7 2024-11-14T07:27:22,313 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into c1fe15ef40484af3ae0ecb57d4a174d7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:22,313 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:22,313 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=12, startTime=1731569241834; duration=0sec 2024-11-14T07:27:22,313 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:22,313 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:22,399 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/5dabe642dc004484b08109dc2b644d39 2024-11-14T07:27:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 is 50, key is test_row_0/C:col10/1731569240577/Put/seqid=0 2024-11-14T07:27:22,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742449_1625 (size=12051) 2024-11-14T07:27:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:22,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:22,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569302786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569302786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569302788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,812 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 2024-11-14T07:27:22,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/f13634efee6b4cdfbaf96c468ff17324 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324 2024-11-14T07:27:22,818 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324, entries=150, sequenceid=131, filesize=30.3 K 2024-11-14T07:27:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/5dabe642dc004484b08109dc2b644d39 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39 2024-11-14T07:27:22,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T07:27:22,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 2024-11-14T07:27:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T07:27:22,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 4687db304ff704d35e9eef31fb5612b9 in 879ms, sequenceid=131, compaction requested=false 2024-11-14T07:27:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T07:27:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T07:27:22,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T07:27:22,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7990 sec 2024-11-14T07:27:22,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8040 sec 2024-11-14T07:27:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:22,898 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:22,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T07:27:22,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111488d005ce2b2a46ffb9ceef42ea78b80b_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:22,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742450_1626 (size=14794) 2024-11-14T07:27:22,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569302905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111488d005ce2b2a46ffb9ceef42ea78b80b_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111488d005ce2b2a46ffb9ceef42ea78b80b_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:22,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/95747da8bd3a4edc87e261ef05ce32d7, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569302909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/95747da8bd3a4edc87e261ef05ce32d7 is 175, key is test_row_0/A:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569302910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742451_1627 (size=39749) 2024-11-14T07:27:22,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/95747da8bd3a4edc87e261ef05ce32d7 2024-11-14T07:27:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/87d6bff8a6eb461cbf4560d2d8194752 is 50, key is test_row_0/B:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:22,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742452_1628 (size=12151) 2024-11-14T07:27:22,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569302985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,988 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:22,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569302994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:22,996 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569303014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569303024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569303024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T07:27:23,175 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:23,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-14T07:27:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:23,177 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:23,178 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:23,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569303224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569303234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569303234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:23,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/87d6bff8a6eb461cbf4560d2d8194752 2024-11-14T07:27:23,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:27:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/74349996f0e74e90aeada966c32a8c62 is 50, key is test_row_0/C:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742453_1629 (size=12151) 2024-11-14T07:27:23,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:27:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:23,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569303534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569303544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569303544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:23,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:27:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:23,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/74349996f0e74e90aeada966c32a8c62 2024-11-14T07:27:23,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/95747da8bd3a4edc87e261ef05ce32d7 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7 2024-11-14T07:27:23,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7, entries=200, sequenceid=161, filesize=38.8 K 2024-11-14T07:27:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/87d6bff8a6eb461cbf4560d2d8194752 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752 2024-11-14T07:27:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752, entries=150, sequenceid=161, filesize=11.9 K 2024-11-14T07:27:23,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/74349996f0e74e90aeada966c32a8c62 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62 2024-11-14T07:27:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62, entries=150, sequenceid=161, filesize=11.9 K 2024-11-14T07:27:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 4687db304ff704d35e9eef31fb5612b9 in 862ms, sequenceid=161, compaction requested=true 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:23,761 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:23,761 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:23,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:23,762 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,762 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,762 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=99.6 K 2024-11-14T07:27:23,762 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/9bff1737522943f3a8e0b46b6bef600a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=35.6 K 2024-11-14T07:27:23,762 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7] 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bff1737522943f3a8e0b46b6bef600a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7225aa61ee94b49bf18b10613e57678, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:23,762 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dabe642dc004484b08109dc2b644d39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731569240577 2024-11-14T07:27:23,763 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d6bff8a6eb461cbf4560d2d8194752, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:23,763 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13634efee6b4cdfbaf96c468ff17324, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731569240577 2024-11-14T07:27:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,763 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95747da8bd3a4edc87e261ef05ce32d7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,768 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,769 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:23,769 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2a507ce02b884be6a0423994fce6cf07 is 50, key is test_row_0/B:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,770 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114878b86506ad24e46b72d5c444d02d7ad_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,772 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114878b86506ad24e46b72d5c444d02d7ad_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:23,773 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114878b86506ad24e46b72d5c444d02d7ad_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742454_1630 (size=12493) 2024-11-14T07:27:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742455_1631 (size=4469) 2024-11-14T07:27:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,786 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#537 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,787 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/27b1e4c8fa724086a60c2fc0f244c694 is 175, key is test_row_0/A:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T07:27:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742456_1632 (size=31447) 2024-11-14T07:27:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T07:27:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ab103e9225d444f9c692352f752b697_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569242907/Put/seqid=0 2024-11-14T07:27:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,798 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/27b1e4c8fa724086a60c2fc0f244c694 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694 2024-11-14T07:27:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,801 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into 27b1e4c8fa724086a60c2fc0f244c694(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:23,801 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:23,802 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569243761; duration=0sec 2024-11-14T07:27:23,802 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:23,802 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:23,802 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:23,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742457_1633 (size=9814) 2024-11-14T07:27:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,803 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:23,803 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:23,803 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:23,803 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/c1fe15ef40484af3ae0ecb57d4a174d7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=35.6 K 2024-11-14T07:27:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,803 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1fe15ef40484af3ae0ecb57d4a174d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731569239936 2024-11-14T07:27:23,803 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a27cd91c7ccb4958acbc3c6c4a1bdca1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731569240577 2024-11-14T07:27:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,804 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74349996f0e74e90aeada966c32a8c62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,807 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ab103e9225d444f9c692352f752b697_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ab103e9225d444f9c692352f752b697_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/94d29017ea21498c84c993404bde0b8e, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/94d29017ea21498c84c993404bde0b8e is 175, key is test_row_0/A:col10/1731569242907/Put/seqid=0 2024-11-14T07:27:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,811 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#539 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:23,811 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/366a1498709f4c178211e185eb19ddb2 is 50, key is test_row_0/C:col10/1731569242897/Put/seqid=0 2024-11-14T07:27:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742458_1634 (size=22461) 2024-11-14T07:27:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742459_1635 (size=12493) 2024-11-14T07:27:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,859 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/366a1498709f4c178211e185eb19ddb2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/366a1498709f4c178211e185eb19ddb2 2024-11-14T07:27:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,862 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into 366a1498709f4c178211e185eb19ddb2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:23,862 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:23,862 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569243761; duration=0sec 2024-11-14T07:27:23,863 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:23,863 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:24,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569304095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569304096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569304096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,181 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2a507ce02b884be6a0423994fce6cf07 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2a507ce02b884be6a0423994fce6cf07 2024-11-14T07:27:24,185 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into 2a507ce02b884be6a0423994fce6cf07(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:24,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:24,185 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569243761; duration=0sec 2024-11-14T07:27:24,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:24,185 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569304204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569304204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569304204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/94d29017ea21498c84c993404bde0b8e 2024-11-14T07:27:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/41490dee88324be1a7f74c05a7da4518 is 50, key is test_row_0/B:col10/1731569242907/Put/seqid=0 2024-11-14T07:27:24,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742460_1636 (size=9757) 2024-11-14T07:27:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569304414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569304414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569304415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,638 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/41490dee88324be1a7f74c05a7da4518 2024-11-14T07:27:24,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b5b081135ea740269b2b01578d65d88d is 50, key is test_row_0/C:col10/1731569242907/Put/seqid=0 2024-11-14T07:27:24,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742461_1637 (size=9757) 2024-11-14T07:27:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569304725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569304725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569304725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b5b081135ea740269b2b01578d65d88d 2024-11-14T07:27:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/94d29017ea21498c84c993404bde0b8e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e 2024-11-14T07:27:25,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e, entries=100, sequenceid=170, filesize=21.9 K 2024-11-14T07:27:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/41490dee88324be1a7f74c05a7da4518 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518 2024-11-14T07:27:25,068 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518, entries=100, sequenceid=170, filesize=9.5 K 2024-11-14T07:27:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b5b081135ea740269b2b01578d65d88d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d 2024-11-14T07:27:25,071 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d, entries=100, sequenceid=170, filesize=9.5 K 2024-11-14T07:27:25,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 4687db304ff704d35e9eef31fb5612b9 in 1282ms, sequenceid=170, compaction requested=false 2024-11-14T07:27:25,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:25,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-14T07:27:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-14T07:27:25,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T07:27:25,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8960 sec 2024-11-14T07:27:25,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.9000 sec 2024-11-14T07:27:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:25,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:25,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T07:27:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569305238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f3fb90fb3e54582b4392df28aa20062_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:25,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569305242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569305243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742462_1638 (size=17284) 2024-11-14T07:27:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T07:27:25,325 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:25,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-14T07:27:25,327 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:25,327 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:25,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569305344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569305354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569305354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:25,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569305554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569305565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569305565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:25,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:25,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141f3fb90fb3e54582b4392df28aa20062_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f3fb90fb3e54582b4392df28aa20062_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:25,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/7f7a052cc6a84d4fb5596b524798473d, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:25,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/7f7a052cc6a84d4fb5596b524798473d is 175, key is test_row_0/A:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:25,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742463_1639 (size=48389) 2024-11-14T07:27:25,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:25,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569305864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569305874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569305874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:25,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:25,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/7f7a052cc6a84d4fb5596b524798473d 2024-11-14T07:27:26,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dbebe565ad84d40b084b67c9c05aa7e is 50, key is test_row_0/B:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742464_1640 (size=12151) 2024-11-14T07:27:26,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569306374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:26,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569306384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569306385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:26,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:26,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dbebe565ad84d40b084b67c9c05aa7e 2024-11-14T07:27:26,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b1dc73c929ac4c2e849e73ef2221b4dd is 50, key is test_row_0/C:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:26,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742465_1641 (size=12151) 2024-11-14T07:27:26,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:26,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b1dc73c929ac4c2e849e73ef2221b4dd 2024-11-14T07:27:26,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/7f7a052cc6a84d4fb5596b524798473d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d 2024-11-14T07:27:26,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d, entries=250, sequenceid=204, filesize=47.3 K 2024-11-14T07:27:26,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/2dbebe565ad84d40b084b67c9c05aa7e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e 2024-11-14T07:27:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T07:27:26,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b1dc73c929ac4c2e849e73ef2221b4dd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 4687db304ff704d35e9eef31fb5612b9 in 1681ms, sequenceid=204, compaction requested=true 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:26,918 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:26,918 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:26,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,919 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:26,919 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,919 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,919 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=99.9 K 2024-11-14T07:27:26,919 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,919 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d] 2024-11-14T07:27:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,919 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:26,919 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,920 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,920 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2a507ce02b884be6a0423994fce6cf07, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=33.6 K 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,920 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b1e4c8fa724086a60c2fc0f244c694, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:26,920 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a507ce02b884be6a0423994fce6cf07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,920 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d29017ea21498c84c993404bde0b8e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731569242907 2024-11-14T07:27:26,920 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 41490dee88324be1a7f74c05a7da4518, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731569242907 2024-11-14T07:27:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,921 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dbebe565ad84d40b084b67c9c05aa7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:26,921 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7a052cc6a84d4fb5596b524798473d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,932 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,933 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:26,934 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cd9371445d3f4b7e8f0d6b6feaf1dcde_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:26,934 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f661f3119e5e4b6c87f870320072251d is 50, key is test_row_0/B:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,935 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cd9371445d3f4b7e8f0d6b6feaf1dcde_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,936 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cd9371445d3f4b7e8f0d6b6feaf1dcde_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742467_1643 (size=4469) 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,969 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#546 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,969 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/4b7eb498abe64cffa9dcb5325a825d8e is 175, key is test_row_0/A:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742466_1642 (size=12595) 2024-11-14T07:27:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,985 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f661f3119e5e4b6c87f870320072251d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f661f3119e5e4b6c87f870320072251d 2024-11-14T07:27:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into f661f3119e5e4b6c87f870320072251d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:26,989 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:26,989 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569246918; duration=0sec 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:26,989 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,990 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:26,990 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,990 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,990 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/366a1498709f4c178211e185eb19ddb2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=33.6 K 2024-11-14T07:27:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,990 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 366a1498709f4c178211e185eb19ddb2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731569242784 2024-11-14T07:27:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b081135ea740269b2b01578d65d88d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731569242907 2024-11-14T07:27:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,991 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b1dc73c929ac4c2e849e73ef2221b4dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,998 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:26,999 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/4359a2edff6a42629c72b67dcf6944ba is 50, key is test_row_0/C:col10/1731569245237/Put/seqid=0 2024-11-14T07:27:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742468_1644 (size=31549) 2024-11-14T07:27:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,009 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/4b7eb498abe64cffa9dcb5325a825d8e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,014 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into 4b7eb498abe64cffa9dcb5325a825d8e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:27,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,014 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569246918; duration=0sec 2024-11-14T07:27:27,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:27,014 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742469_1645 (size=12595) 2024-11-14T07:27:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,020 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/4359a2edff6a42629c72b67dcf6944ba as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/4359a2edff6a42629c72b67dcf6944ba 2024-11-14T07:27:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into 4359a2edff6a42629c72b67dcf6944ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:27,032 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:27,032 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569246918; duration=0sec 2024-11-14T07:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,032 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:27,033 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T07:27:27,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:27,038 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-14T07:27:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145fc9f4a63ccc4368ade34aff55a9e2db_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569245241/Put/seqid=0 2024-11-14T07:27:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742470_1646 (size=7324) 2024-11-14T07:27:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:27,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145fc9f4a63ccc4368ade34aff55a9e2db_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145fc9f4a63ccc4368ade34aff55a9e2db_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:27,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/c18819c84b2f44fcafbba629c3ecbb1d, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:27,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/c18819c84b2f44fcafbba629c3ecbb1d is 175, key is test_row_0/A:col10/1731569245241/Put/seqid=0 2024-11-14T07:27:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742471_1647 (size=13815) 2024-11-14T07:27:27,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/c18819c84b2f44fcafbba629c3ecbb1d 2024-11-14T07:27:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d66edb3db0f8427e958410ce81b73d50 is 50, key is test_row_0/B:col10/1731569245241/Put/seqid=0 2024-11-14T07:27:27,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569307497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569307505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569307505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742472_1648 (size=7365) 2024-11-14T07:27:27,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d66edb3db0f8427e958410ce81b73d50 2024-11-14T07:27:27,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/95d376f6f4f44332bb5db66e82d8bd40 is 50, key is test_row_0/C:col10/1731569245241/Put/seqid=0 2024-11-14T07:27:27,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742473_1649 (size=7365) 2024-11-14T07:27:27,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569307614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569307614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569307614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569307824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569307824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569307824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:27,926 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/95d376f6f4f44332bb5db66e82d8bd40 2024-11-14T07:27:27,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/c18819c84b2f44fcafbba629c3ecbb1d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d 2024-11-14T07:27:27,933 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d, entries=50, sequenceid=212, filesize=13.5 K 2024-11-14T07:27:27,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d66edb3db0f8427e958410ce81b73d50 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50 2024-11-14T07:27:27,937 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50, entries=50, sequenceid=212, filesize=7.2 K 2024-11-14T07:27:27,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/95d376f6f4f44332bb5db66e82d8bd40 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40 2024-11-14T07:27:27,940 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40, entries=50, sequenceid=212, filesize=7.2 K 2024-11-14T07:27:27,940 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=194.56 KB/199230 for 4687db304ff704d35e9eef31fb5612b9 in 903ms, sequenceid=212, compaction requested=false 2024-11-14T07:27:27,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-14T07:27:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-14T07:27:27,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T07:27:27,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6150 sec 2024-11-14T07:27:27,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.6180 sec 2024-11-14T07:27:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:28,137 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:28,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-14T07:27:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569308135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569308135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569308138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111480239daf3f16493c8e73fee9bc28b612_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742474_1650 (size=12304) 2024-11-14T07:27:28,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:28,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111480239daf3f16493c8e73fee9bc28b612_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480239daf3f16493c8e73fee9bc28b612_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:28,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/844280e953d246f792436d56efeddbca, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:28,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/844280e953d246f792436d56efeddbca is 175, key is test_row_0/A:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:28,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742475_1651 (size=31105) 2024-11-14T07:27:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569308244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569308455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/844280e953d246f792436d56efeddbca 2024-11-14T07:27:28,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d023d60af39c486f8f0dbf8c2102d55d is 50, key is test_row_0/B:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:28,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742476_1652 (size=12151) 2024-11-14T07:27:28,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d023d60af39c486f8f0dbf8c2102d55d 2024-11-14T07:27:28,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 is 50, key is test_row_0/C:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:28,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742477_1653 (size=12151) 2024-11-14T07:27:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569308644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569308644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569308764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 2024-11-14T07:27:29,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/844280e953d246f792436d56efeddbca as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca 2024-11-14T07:27:29,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca, entries=150, sequenceid=245, filesize=30.4 K 2024-11-14T07:27:29,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d023d60af39c486f8f0dbf8c2102d55d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d 2024-11-14T07:27:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T07:27:29,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1, entries=150, sequenceid=245, filesize=11.9 K 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 4687db304ff704d35e9eef31fb5612b9 in 893ms, sequenceid=245, compaction requested=true 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:29,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:29,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,030 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:29,031 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:29,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 76469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,031 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=74.7 K 2024-11-14T07:27:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,031 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,031 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca] 2024-11-14T07:27:29,032 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:29,032 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,032 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,032 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f661f3119e5e4b6c87f870320072251d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=31.4 K 2024-11-14T07:27:29,032 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7eb498abe64cffa9dcb5325a825d8e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,032 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f661f3119e5e4b6c87f870320072251d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,032 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18819c84b2f44fcafbba629c3ecbb1d, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569245241 2024-11-14T07:27:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d66edb3db0f8427e958410ce81b73d50, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569245241 2024-11-14T07:27:29,033 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844280e953d246f792436d56efeddbca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting d023d60af39c486f8f0dbf8c2102d55d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,040 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/cbd0f9871481473f8770f47e9727faea is 50, key is test_row_0/B:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,045 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111438bcd79c931c4515a0314559b88768d4_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,046 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111438bcd79c931c4515a0314559b88768d4_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111438bcd79c931c4515a0314559b88768d4_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742478_1654 (size=12697) 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,085 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/cbd0f9871481473f8770f47e9727faea as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/cbd0f9871481473f8770f47e9727faea 2024-11-14T07:27:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into cbd0f9871481473f8770f47e9727faea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:29,089 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569249030; duration=0sec 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:29,089 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:29,089 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:29,090 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:29,090 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/4359a2edff6a42629c72b67dcf6944ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=31.4 K 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4359a2edff6a42629c72b67dcf6944ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731569244093 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d376f6f4f44332bb5db66e82d8bd40, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731569245241 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,090 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting e34f5cc1bfcf4f4f9dc30c61ad5038d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742479_1655 (size=4469) 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#555 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/4ffe0af320e043e5aef1f02a4ef74069 is 175, key is test_row_0/A:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#556 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/765f15fc772643369e35a2dad7ed9f91 is 50, key is test_row_0/C:col10/1731569248136/Put/seqid=0 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742480_1656 (size=31651) 2024-11-14T07:27:29,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742481_1657 (size=12697) 2024-11-14T07:27:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/765f15fc772643369e35a2dad7ed9f91 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/765f15fc772643369e35a2dad7ed9f91 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into 765f15fc772643369e35a2dad7ed9f91(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:29,129 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569249031; duration=0sec 2024-11-14T07:27:29,129 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:29,129 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:29,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:29,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T07:27:29,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114646d63e99f3e45d5be4177b9fbfb8cf1_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569249318/Put/seqid=0 2024-11-14T07:27:29,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742482_1658 (size=12404) 2024-11-14T07:27:29,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:29,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114646d63e99f3e45d5be4177b9fbfb8cf1_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114646d63e99f3e45d5be4177b9fbfb8cf1_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:29,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/dd127c5ff677400ab70fef00de3c3728, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:29,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/dd127c5ff677400ab70fef00de3c3728 is 175, key is test_row_0/A:col10/1731569249318/Put/seqid=0 2024-11-14T07:27:29,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742483_1659 (size=31205) 2024-11-14T07:27:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569309477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T07:27:29,494 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:29,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-14T07:27:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:29,497 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:29,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:29,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:29,523 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/4ffe0af320e043e5aef1f02a4ef74069 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069 2024-11-14T07:27:29,527 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into 4ffe0af320e043e5aef1f02a4ef74069(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:29,527 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:29,527 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569249030; duration=0sec 2024-11-14T07:27:29,527 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:29,528 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569309585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:29,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569309655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:29,657 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569309665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:29,666 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569309795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:29,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/dd127c5ff677400ab70fef00de3c3728 2024-11-14T07:27:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:29,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/84a5c12a8555445783343221a44ddb30 is 50, key is test_row_0/B:col10/1731569249318/Put/seqid=0 2024-11-14T07:27:29,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742484_1660 (size=12251) 2024-11-14T07:27:29,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/84a5c12a8555445783343221a44ddb30 2024-11-14T07:27:29,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a812b7dd0fd649cbbe99911b5bbd8148 is 50, key is test_row_0/C:col10/1731569249318/Put/seqid=0 2024-11-14T07:27:29,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742485_1661 (size=12251) 2024-11-14T07:27:29,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:29,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569310104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:30,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:30,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a812b7dd0fd649cbbe99911b5bbd8148 2024-11-14T07:27:30,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/dd127c5ff677400ab70fef00de3c3728 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728 2024-11-14T07:27:30,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728, entries=150, sequenceid=258, filesize=30.5 K 2024-11-14T07:27:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/84a5c12a8555445783343221a44ddb30 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30 2024-11-14T07:27:30,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T07:27:30,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a812b7dd0fd649cbbe99911b5bbd8148 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148 2024-11-14T07:27:30,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:30,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T07:27:30,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4687db304ff704d35e9eef31fb5612b9 in 915ms, sequenceid=258, compaction requested=false 2024-11-14T07:27:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:30,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T07:27:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T07:27:30,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412fd109e694245338207a9fa1733db10_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569249476/Put/seqid=0 2024-11-14T07:27:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742486_1662 (size=12454) 2024-11-14T07:27:30,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:30,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111412fd109e694245338207a9fa1733db10_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412fd109e694245338207a9fa1733db10_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/8ebfe4dccca8495f96f6356e3da2fe78, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:30,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/8ebfe4dccca8495f96f6356e3da2fe78 is 175, key is test_row_0/A:col10/1731569249476/Put/seqid=0 2024-11-14T07:27:30,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742487_1663 (size=31255) 2024-11-14T07:27:30,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/8ebfe4dccca8495f96f6356e3da2fe78 2024-11-14T07:27:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f347a16d62154d628054d94b28762f09 is 50, key is test_row_0/B:col10/1731569249476/Put/seqid=0 2024-11-14T07:27:30,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742488_1664 (size=12301) 2024-11-14T07:27:30,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f347a16d62154d628054d94b28762f09 2024-11-14T07:27:30,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/0344a6a1952d4d79a6e309fe5e19bd9b is 50, key is test_row_0/C:col10/1731569249476/Put/seqid=0 2024-11-14T07:27:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742489_1665 (size=12301) 2024-11-14T07:27:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:30,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569310643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569310754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:30,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/0344a6a1952d4d79a6e309fe5e19bd9b 2024-11-14T07:27:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/8ebfe4dccca8495f96f6356e3da2fe78 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78 2024-11-14T07:27:30,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78, entries=150, sequenceid=284, filesize=30.5 K 2024-11-14T07:27:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/f347a16d62154d628054d94b28762f09 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09 2024-11-14T07:27:30,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T07:27:30,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/0344a6a1952d4d79a6e309fe5e19bd9b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b 2024-11-14T07:27:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T07:27:30,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4687db304ff704d35e9eef31fb5612b9 in 460ms, sequenceid=284, compaction requested=true 2024-11-14T07:27:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-14T07:27:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-14T07:27:30,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-14T07:27:30,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3770 sec 2024-11-14T07:27:30,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.3820 sec 2024-11-14T07:27:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:30,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:30,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T07:27:30,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141fad67693bc74aa49b4559aaadea8aea_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:30,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742490_1666 (size=14994) 2024-11-14T07:27:30,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:30,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141fad67693bc74aa49b4559aaadea8aea_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141fad67693bc74aa49b4559aaadea8aea_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:30,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a3043eeef35a4ebbb5566fc47e5c51e9, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:30,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a3043eeef35a4ebbb5566fc47e5c51e9 is 175, key is test_row_0/A:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:30,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742491_1667 (size=39949) 2024-11-14T07:27:30,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a3043eeef35a4ebbb5566fc47e5c51e9 2024-11-14T07:27:30,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/0f77baadfe4145f3b197fcc5190121a9 is 50, key is test_row_0/B:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:30,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742492_1668 (size=12301) 2024-11-14T07:27:31,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569311080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569311194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/0f77baadfe4145f3b197fcc5190121a9 2024-11-14T07:27:31,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/8fba9d79921a4eada990971d0190c2e9 is 50, key is test_row_0/C:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:31,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742493_1669 (size=12301) 2024-11-14T07:27:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569311404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T07:27:31,644 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:31,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-14T07:27:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:31,646 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:31,646 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:31,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:31,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569311664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,668 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569311694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,698 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569311715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:31,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-14T07:27:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:31,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/8fba9d79921a4eada990971d0190c2e9 2024-11-14T07:27:31,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a3043eeef35a4ebbb5566fc47e5c51e9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9 2024-11-14T07:27:31,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9, entries=200, sequenceid=296, filesize=39.0 K 2024-11-14T07:27:31,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/0f77baadfe4145f3b197fcc5190121a9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9 2024-11-14T07:27:31,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9, entries=150, sequenceid=296, filesize=12.0 K 2024-11-14T07:27:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/8fba9d79921a4eada990971d0190c2e9 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9 2024-11-14T07:27:31,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9, entries=150, sequenceid=296, filesize=12.0 K 2024-11-14T07:27:31,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4687db304ff704d35e9eef31fb5612b9 in 852ms, sequenceid=296, compaction requested=true 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:31,819 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:31,819 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:31,820 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:31,820 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:31,820 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,820 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/cbd0f9871481473f8770f47e9727faea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=48.4 K 2024-11-14T07:27:31,820 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd0f9871481473f8770f47e9727faea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134060 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a5c12a8555445783343221a44ddb30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569249318 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:31,821 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,821 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=130.9 K 2024-11-14T07:27:31,821 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting f347a16d62154d628054d94b28762f09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731569249456 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9] 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f77baadfe4145f3b197fcc5190121a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffe0af320e043e5aef1f02a4ef74069, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:31,821 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd127c5ff677400ab70fef00de3c3728, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569249318 2024-11-14T07:27:31,822 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebfe4dccca8495f96f6356e3da2fe78, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731569249456 2024-11-14T07:27:31,822 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3043eeef35a4ebbb5566fc47e5c51e9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:31,826 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:31,827 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114665b372c7c2a4af89e524bdd2f3411ee_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:31,828 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#566 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:31,829 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/0c4acdd249b3484fa932f1c987a7512f is 50, key is test_row_0/B:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:31,829 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114665b372c7c2a4af89e524bdd2f3411ee_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:31,829 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114665b372c7c2a4af89e524bdd2f3411ee_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742494_1670 (size=12983) 2024-11-14T07:27:31,837 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/0c4acdd249b3484fa932f1c987a7512f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0c4acdd249b3484fa932f1c987a7512f 2024-11-14T07:27:31,841 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into 0c4acdd249b3484fa932f1c987a7512f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:31,841 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:31,841 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=12, startTime=1731569251819; duration=0sec 2024-11-14T07:27:31,841 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:31,841 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:31,841 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T07:27:31,842 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T07:27:31,842 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:31,842 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,842 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/765f15fc772643369e35a2dad7ed9f91, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=48.4 K 2024-11-14T07:27:31,842 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 765f15fc772643369e35a2dad7ed9f91, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731569247496 2024-11-14T07:27:31,843 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a812b7dd0fd649cbbe99911b5bbd8148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731569249318 2024-11-14T07:27:31,843 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0344a6a1952d4d79a6e309fe5e19bd9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731569249456 2024-11-14T07:27:31,843 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fba9d79921a4eada990971d0190c2e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:31,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742495_1671 (size=4469) 2024-11-14T07:27:31,848 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#567 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:31,849 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a36ea6f4fc554019aa48b7461a328bfb is 175, key is test_row_0/A:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:31,851 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#568 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:31,851 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/41a6419a5fb74eb384b8d78991d74532 is 50, key is test_row_0/C:col10/1731569250627/Put/seqid=0 2024-11-14T07:27:31,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742496_1672 (size=31937) 2024-11-14T07:27:31,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742497_1673 (size=12983) 2024-11-14T07:27:31,863 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/a36ea6f4fc554019aa48b7461a328bfb as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb 2024-11-14T07:27:31,864 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/41a6419a5fb74eb384b8d78991d74532 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/41a6419a5fb74eb384b8d78991d74532 2024-11-14T07:27:31,867 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into 41a6419a5fb74eb384b8d78991d74532(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:31,867 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into a36ea6f4fc554019aa48b7461a328bfb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:31,867 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=12, startTime=1731569251819; duration=0sec 2024-11-14T07:27:31,867 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=12, startTime=1731569251819; duration=0sec 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:31,867 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:31,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-14T07:27:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:31,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T07:27:31,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111479d7e843a7794f70ba88d37ddfe35db8_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569251079/Put/seqid=0 2024-11-14T07:27:31,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742498_1674 (size=12454) 2024-11-14T07:27:31,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111479d7e843a7794f70ba88d37ddfe35db8_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111479d7e843a7794f70ba88d37ddfe35db8_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9b39404d546247b899976155daa76bd1, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9b39404d546247b899976155daa76bd1 is 175, key is test_row_0/A:col10/1731569251079/Put/seqid=0 2024-11-14T07:27:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:31,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742499_1675 (size=31255) 2024-11-14T07:27:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569312253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:32,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569312364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:32,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9b39404d546247b899976155daa76bd1 2024-11-14T07:27:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/4e078c701e414f17b2ea213d9bd8e366 is 50, key is test_row_0/B:col10/1731569251079/Put/seqid=0 2024-11-14T07:27:32,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742500_1676 (size=12301) 2024-11-14T07:27:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569312574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:32,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/4e078c701e414f17b2ea213d9bd8e366 2024-11-14T07:27:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b9fe78fe986c4373a4fda8496dcc750a is 50, key is test_row_0/C:col10/1731569251079/Put/seqid=0 2024-11-14T07:27:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:32,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742501_1677 (size=12301) 2024-11-14T07:27:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569312884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45254 deadline: 1731569313065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:33,066 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_1' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45266 deadline: 1731569313084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:33,087 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b9fe78fe986c4373a4fda8496dcc750a 2024-11-14T07:27:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/9b39404d546247b899976155daa76bd1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1 2024-11-14T07:27:33,196 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1, entries=150, sequenceid=323, filesize=30.5 K 2024-11-14T07:27:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/4e078c701e414f17b2ea213d9bd8e366 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366 2024-11-14T07:27:33,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T07:27:33,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b9fe78fe986c4373a4fda8496dcc750a as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a 2024-11-14T07:27:33,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T07:27:33,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4687db304ff704d35e9eef31fb5612b9 in 1254ms, sequenceid=323, compaction requested=false 2024-11-14T07:27:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-14T07:27:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-14T07:27:33,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-14T07:27:33,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-14T07:27:33,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.5630 sec 2024-11-14T07:27:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:33,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:33,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T07:27:33,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e94854655c4c4d0b85528b20f4f344ac_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:33,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742502_1678 (size=12454) 2024-11-14T07:27:33,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:33,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e94854655c4c4d0b85528b20f4f344ac_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e94854655c4c4d0b85528b20f4f344ac_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:33,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/abc2242399394141b4a8b0171d3d467d, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:33,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/abc2242399394141b4a8b0171d3d467d is 175, key is test_row_0/A:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:33,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742503_1679 (size=31255) 2024-11-14T07:27:33,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569313483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569313594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T07:27:33,795 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:33,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T07:27:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-14T07:27:33,797 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T07:27:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:33,798 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T07:27:33,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T07:27:33,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569313804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:33,846 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/abc2242399394141b4a8b0171d3d467d 2024-11-14T07:27:33,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/75efbdc07afa4d4daf1333006fecbc6e is 50, key is test_row_0/B:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:33,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742504_1680 (size=12301) 2024-11-14T07:27:33,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/75efbdc07afa4d4daf1333006fecbc6e 2024-11-14T07:27:33,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a51a43b5977b4effb2bf990b7f44bb7c is 50, key is test_row_0/C:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:33,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742505_1681 (size=12301) 2024-11-14T07:27:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:33,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-14T07:27:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:33,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-14T07:27:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:34,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569314114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:34,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-14T07:27:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T07:27:34,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a51a43b5977b4effb2bf990b7f44bb7c 2024-11-14T07:27:34,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/abc2242399394141b4a8b0171d3d467d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d 2024-11-14T07:27:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d, entries=150, sequenceid=336, filesize=30.5 K 2024-11-14T07:27:34,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/75efbdc07afa4d4daf1333006fecbc6e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e 2024-11-14T07:27:34,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T07:27:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/a51a43b5977b4effb2bf990b7f44bb7c as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c 2024-11-14T07:27:34,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T07:27:34,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4687db304ff704d35e9eef31fb5612b9 in 914ms, sequenceid=336, compaction requested=true 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:34,311 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:34,311 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:34,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:34,312 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:34,312 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,312 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=92.2 K 2024-11-14T07:27:34,312 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0c4acdd249b3484fa932f1c987a7512f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=36.7 K 2024-11-14T07:27:34,312 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d] 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c4acdd249b3484fa932f1c987a7512f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36ea6f4fc554019aa48b7461a328bfb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e078c701e414f17b2ea213d9bd8e366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731569251044 2024-11-14T07:27:34,312 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b39404d546247b899976155daa76bd1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731569251044 2024-11-14T07:27:34,313 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 75efbdc07afa4d4daf1333006fecbc6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:34,313 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting abc2242399394141b4a8b0171d3d467d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:34,317 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:34,318 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cb6faab0f15c48909e374c89c77d3987_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:34,318 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:34,318 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/7589c906370748de8bc2f3f1a8b27afd is 50, key is test_row_0/B:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:34,320 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cb6faab0f15c48909e374c89c77d3987_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:34,320 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb6faab0f15c48909e374c89c77d3987_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:34,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742506_1682 (size=13085) 2024-11-14T07:27:34,329 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/7589c906370748de8bc2f3f1a8b27afd as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7589c906370748de8bc2f3f1a8b27afd 2024-11-14T07:27:34,332 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into 7589c906370748de8bc2f3f1a8b27afd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:34,332 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:34,332 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569254311; duration=0sec 2024-11-14T07:27:34,333 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:34,333 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:34,333 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:34,334 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:34,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742507_1683 (size=4469) 2024-11-14T07:27:34,334 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:34,334 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,334 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/41a6419a5fb74eb384b8d78991d74532, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=36.7 K 2024-11-14T07:27:34,334 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a6419a5fb74eb384b8d78991d74532, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731569250627 2024-11-14T07:27:34,335 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b9fe78fe986c4373a4fda8496dcc750a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731569251044 2024-11-14T07:27:34,336 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#576 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:34,336 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting a51a43b5977b4effb2bf990b7f44bb7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:34,336 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/695307d67770400eb3d3ac1586eea7ed is 175, key is test_row_0/A:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:34,346 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:34,347 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b8eb62995a3f43caa5a361311cb272f8 is 50, key is test_row_0/C:col10/1731569252245/Put/seqid=0 2024-11-14T07:27:34,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742508_1684 (size=32039) 2024-11-14T07:27:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742509_1685 (size=13085) 2024-11-14T07:27:34,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-14T07:27:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:34,406 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T07:27:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146974f894ec234673a00d67aa24182dcd_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569253462/Put/seqid=0 2024-11-14T07:27:34,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742510_1686 (size=12454) 2024-11-14T07:27:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. as already flushing 2024-11-14T07:27:34,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569314665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:34,706 INFO [Thread-2620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2620. 2024-11-14T07:27:34,706 DEBUG [Thread-2620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:34,706 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,706 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:34,706 INFO [Thread-2612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2612. 2024-11-14T07:27:34,706 DEBUG [Thread-2612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:34,706 DEBUG [Thread-2612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,706 DEBUG [Thread-2612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:34,707 INFO [Thread-2616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2616. 2024-11-14T07:27:34,707 DEBUG [Thread-2616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:34,707 DEBUG [Thread-2616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,707 DEBUG [Thread-2616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:34,710 INFO [Thread-2618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2618. 2024-11-14T07:27:34,710 INFO [Thread-2614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2614. 2024-11-14T07:27:34,710 DEBUG [Thread-2614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:34,710 DEBUG [Thread-2618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:34,710 DEBUG [Thread-2614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,710 DEBUG [Thread-2618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,710 DEBUG [Thread-2614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,710 DEBUG [Thread-2618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:34,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:34,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:34,752 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/695307d67770400eb3d3ac1586eea7ed as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed 2024-11-14T07:27:34,755 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into 695307d67770400eb3d3ac1586eea7ed(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:34,755 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:34,755 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569254311; duration=0sec 2024-11-14T07:27:34,755 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:34,755 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:34,764 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/b8eb62995a3f43caa5a361311cb272f8 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b8eb62995a3f43caa5a361311cb272f8 2024-11-14T07:27:34,768 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into b8eb62995a3f43caa5a361311cb272f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:34,768 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:34,768 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569254311; duration=0sec 2024-11-14T07:27:34,768 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:34,768 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:34,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569314774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:34,822 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146974f894ec234673a00d67aa24182dcd_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146974f894ec234673a00d67aa24182dcd_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:34,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/2a0f621966ae43338a1476ccab61df8f, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:34,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/2a0f621966ae43338a1476ccab61df8f is 175, key is test_row_0/A:col10/1731569253462/Put/seqid=0 2024-11-14T07:27:34,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742511_1687 (size=31255) 2024-11-14T07:27:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:34,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569314984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:35,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/2a0f621966ae43338a1476ccab61df8f 2024-11-14T07:27:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/707f391266d4462ca76001f6b2396575 is 50, key is test_row_0/B:col10/1731569253462/Put/seqid=0 2024-11-14T07:27:35,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742512_1688 (size=12301) 2024-11-14T07:27:35,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569315295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:35,635 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/707f391266d4462ca76001f6b2396575 2024-11-14T07:27:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/5f8818260abc49e38e3d2fe74370b1af is 50, key is test_row_0/C:col10/1731569253462/Put/seqid=0 2024-11-14T07:27:35,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742513_1689 (size=12301) 2024-11-14T07:27:35,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45204 deadline: 1731569315705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:35,706 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_0' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45230 deadline: 1731569315714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:35,715 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c7aa6e2790b3,46235,1731569085523 for 'test_row_2' in 4687db304ff704d35e9eef31fb5612b9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T07:27:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T07:27:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45196 deadline: 1731569315805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 , the old value is region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T07:27:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., hostname=c7aa6e2790b3,46235,1731569085523, seqNum=5 because the exception is null or not the one we care about 2024-11-14T07:27:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:36,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/5f8818260abc49e38e3d2fe74370b1af 2024-11-14T07:27:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/2a0f621966ae43338a1476ccab61df8f as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f 2024-11-14T07:27:36,050 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f, entries=150, sequenceid=360, filesize=30.5 K 2024-11-14T07:27:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/707f391266d4462ca76001f6b2396575 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575 2024-11-14T07:27:36,054 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T07:27:36,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/5f8818260abc49e38e3d2fe74370b1af as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af 2024-11-14T07:27:36,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af, entries=150, sequenceid=360, filesize=12.0 K 2024-11-14T07:27:36,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4687db304ff704d35e9eef31fb5612b9 in 1651ms, sequenceid=360, compaction requested=false 2024-11-14T07:27:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c7aa6e2790b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-14T07:27:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-14T07:27:36,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-14T07:27:36,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2610 sec 2024-11-14T07:27:36,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.2650 sec 2024-11-14T07:27:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46235 {}] regionserver.HRegion(8855): Flush requested on 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:36,816 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T07:27:36,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T07:27:36,816 INFO [Thread-2609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2609. 2024-11-14T07:27:36,816 DEBUG [Thread-2609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:36,816 DEBUG [Thread-2609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:36,816 DEBUG [Thread-2609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:36,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:36,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146f476a0c921249f3b039eea1b353b365_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:36,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742514_1690 (size=12454) 2024-11-14T07:27:37,225 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:37,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146f476a0c921249f3b039eea1b353b365_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f476a0c921249f3b039eea1b353b365_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:37,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/5acb4d760c54485daf9fb9626280c19e, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:37,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/5acb4d760c54485daf9fb9626280c19e is 175, key is test_row_0/A:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:37,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742515_1691 (size=31255) 2024-11-14T07:27:37,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/5acb4d760c54485daf9fb9626280c19e 2024-11-14T07:27:37,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/fa1e420b6bfb460fa713d07b37d58af3 is 50, key is test_row_0/B:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:37,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742516_1692 (size=12301) 2024-11-14T07:27:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-14T07:27:37,974 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:38,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/fa1e420b6bfb460fa713d07b37d58af3 2024-11-14T07:27:38,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/00144180bd17468e9059c2d6551db414 is 50, key is test_row_0/C:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:38,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742517_1693 (size=12301) 2024-11-14T07:27:38,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/00144180bd17468e9059c2d6551db414 2024-11-14T07:27:38,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/5acb4d760c54485daf9fb9626280c19e as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e 2024-11-14T07:27:38,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e, entries=150, sequenceid=376, filesize=30.5 K 2024-11-14T07:27:38,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/fa1e420b6bfb460fa713d07b37d58af3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3 2024-11-14T07:27:38,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T07:27:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/00144180bd17468e9059c2d6551db414 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414 2024-11-14T07:27:38,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T07:27:38,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 4687db304ff704d35e9eef31fb5612b9 in 1649ms, sequenceid=376, compaction requested=true 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4687db304ff704d35e9eef31fb5612b9:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:38,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/A is initiating minor compaction (all files) 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/B is initiating minor compaction (all files) 2024-11-14T07:27:38,466 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/A in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:38,466 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/B in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:38,466 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=92.3 K 2024-11-14T07:27:38,466 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7589c906370748de8bc2f3f1a8b27afd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=36.8 K 2024-11-14T07:27:38,466 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:38,466 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. files: [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e] 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 7589c906370748de8bc2f3f1a8b27afd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 695307d67770400eb3d3ac1586eea7ed, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 707f391266d4462ca76001f6b2396575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731569253462 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0f621966ae43338a1476ccab61df8f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731569253462 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1e420b6bfb460fa713d07b37d58af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731569254649 2024-11-14T07:27:38,467 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5acb4d760c54485daf9fb9626280c19e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731569254649 2024-11-14T07:27:38,471 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:38,471 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#B#compaction#584 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:38,472 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d31e8a011a5e41baa762c7ea4c133351 is 50, key is test_row_0/B:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:38,473 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e04929de67c84cea93b5c0465988e0e6_4687db304ff704d35e9eef31fb5612b9 store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742518_1694 (size=13187) 2024-11-14T07:27:38,476 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e04929de67c84cea93b5c0465988e0e6_4687db304ff704d35e9eef31fb5612b9, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:38,476 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e04929de67c84cea93b5c0465988e0e6_4687db304ff704d35e9eef31fb5612b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:38,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742519_1695 (size=4469) 2024-11-14T07:27:38,879 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#A#compaction#585 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:38,879 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/af3d5280de7f4004966d6bd3436bf670 is 175, key is test_row_0/A:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:38,880 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/d31e8a011a5e41baa762c7ea4c133351 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d31e8a011a5e41baa762c7ea4c133351 2024-11-14T07:27:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742520_1696 (size=32141) 2024-11-14T07:27:38,884 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/B of 4687db304ff704d35e9eef31fb5612b9 into d31e8a011a5e41baa762c7ea4c133351(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:38,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:38,884 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/B, priority=13, startTime=1731569258466; duration=0sec 2024-11-14T07:27:38,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T07:27:38,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:B 2024-11-14T07:27:38,884 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T07:27:38,885 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T07:27:38,885 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1541): 4687db304ff704d35e9eef31fb5612b9/C is initiating minor compaction (all files) 2024-11-14T07:27:38,885 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4687db304ff704d35e9eef31fb5612b9/C in TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:38,885 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b8eb62995a3f43caa5a361311cb272f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414] into tmpdir=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp, totalSize=36.8 K 2024-11-14T07:27:38,885 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting b8eb62995a3f43caa5a361311cb272f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731569252242 2024-11-14T07:27:38,886 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8818260abc49e38e3d2fe74370b1af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731569253462 2024-11-14T07:27:38,886 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] compactions.Compactor(225): Compacting 00144180bd17468e9059c2d6551db414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731569254649 2024-11-14T07:27:38,892 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4687db304ff704d35e9eef31fb5612b9#C#compaction#586 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T07:27:38,893 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/5f1b67f7417d4f748cd5b3768479fe8b is 50, key is test_row_0/C:col10/1731569254649/Put/seqid=0 2024-11-14T07:27:38,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742521_1697 (size=13187) 2024-11-14T07:27:39,286 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/af3d5280de7f4004966d6bd3436bf670 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/af3d5280de7f4004966d6bd3436bf670 2024-11-14T07:27:39,289 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/A of 4687db304ff704d35e9eef31fb5612b9 into af3d5280de7f4004966d6bd3436bf670(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:39,289 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:39,290 INFO [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/A, priority=13, startTime=1731569258466; duration=0sec 2024-11-14T07:27:39,290 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:39,290 DEBUG [RS:0;c7aa6e2790b3:46235-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:A 2024-11-14T07:27:39,299 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/5f1b67f7417d4f748cd5b3768479fe8b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f1b67f7417d4f748cd5b3768479fe8b 2024-11-14T07:27:39,302 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4687db304ff704d35e9eef31fb5612b9/C of 4687db304ff704d35e9eef31fb5612b9 into 5f1b67f7417d4f748cd5b3768479fe8b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T07:27:39,302 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4687db304ff704d35e9eef31fb5612b9: 2024-11-14T07:27:39,302 INFO [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., storeName=4687db304ff704d35e9eef31fb5612b9/C, priority=13, startTime=1731569258466; duration=0sec 2024-11-14T07:27:39,302 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T07:27:39,302 DEBUG [RS:0;c7aa6e2790b3:46235-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4687db304ff704d35e9eef31fb5612b9:C 2024-11-14T07:27:43,106 INFO [Thread-2607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2607. 2024-11-14T07:27:43,106 DEBUG [Thread-2607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:43,106 DEBUG [Thread-2607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:43,106 DEBUG [Thread-2607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:43,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:43,116 INFO [Thread-2605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2605. 2024-11-14T07:27:43,116 DEBUG [Thread-2605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:43,116 DEBUG [Thread-2605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:43,116 DEBUG [Thread-2605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:43,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:43,897 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T07:27:45,755 INFO [Thread-2601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2601. 2024-11-14T07:27:45,755 DEBUG [Thread-2601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:45,755 DEBUG [Thread-2601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,755 DEBUG [Thread-2601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:45,816 INFO [Thread-2603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2603. 2024-11-14T07:27:45,816 DEBUG [Thread-2603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T07:27:45,816 DEBUG [Thread-2603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,816 DEBUG [Thread-2603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4278 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4161 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4275 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4279 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4135 2024-11-14T07:27:45,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T07:27:45,817 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:27:45,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:27:45,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:45,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:45,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T07:27:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:45,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569265820"}]},"ts":"1731569265820"} 2024-11-14T07:27:45,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T07:27:45,872 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T07:27:45,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T07:27:45,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, UNASSIGN}] 2024-11-14T07:27:45,876 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, UNASSIGN 2024-11-14T07:27:45,877 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=CLOSING, regionLocation=c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:45,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, UNASSIGN because future has completed 2024-11-14T07:27:45,879 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T07:27:45,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523}] 2024-11-14T07:27:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:46,032 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:46,032 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T07:27:46,032 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing 4687db304ff704d35e9eef31fb5612b9, disabling compactions & flushes 2024-11-14T07:27:46,032 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:46,032 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:46,032 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. after waiting 0 ms 2024-11-14T07:27:46,032 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:46,032 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing 4687db304ff704d35e9eef31fb5612b9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T07:27:46,037 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144bf713e6f6824eae82bb779e737cf9e0_4687db304ff704d35e9eef31fb5612b9 is 50, key is test_row_0/A:col10/1731569265754/Put/seqid=0 2024-11-14T07:27:46,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742522_1698 (size=12454) 2024-11-14T07:27:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:46,441 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T07:27:46,444 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144bf713e6f6824eae82bb779e737cf9e0_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bf713e6f6824eae82bb779e737cf9e0_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:46,444 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/82db3c6f24d74946b390a4d7b1d7b290, store: [table=TestAcidGuarantees family=A region=4687db304ff704d35e9eef31fb5612b9] 2024-11-14T07:27:46,445 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/82db3c6f24d74946b390a4d7b1d7b290 is 175, key is test_row_0/A:col10/1731569265754/Put/seqid=0 2024-11-14T07:27:46,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742523_1699 (size=31255) 2024-11-14T07:27:46,848 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/82db3c6f24d74946b390a4d7b1d7b290 2024-11-14T07:27:46,853 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/bfa6fc27a6b0415e82227431925f9ba2 is 50, key is test_row_0/B:col10/1731569265754/Put/seqid=0 2024-11-14T07:27:46,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742524_1700 (size=12301) 2024-11-14T07:27:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:47,257 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/bfa6fc27a6b0415e82227431925f9ba2 2024-11-14T07:27:47,262 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/19d52ad6512047d68ef11a042e466676 is 50, key is test_row_0/C:col10/1731569265754/Put/seqid=0 2024-11-14T07:27:47,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742525_1701 (size=12301) 2024-11-14T07:27:47,665 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/19d52ad6512047d68ef11a042e466676 2024-11-14T07:27:47,668 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/A/82db3c6f24d74946b390a4d7b1d7b290 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/82db3c6f24d74946b390a4d7b1d7b290 2024-11-14T07:27:47,671 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/82db3c6f24d74946b390a4d7b1d7b290, entries=150, sequenceid=386, filesize=30.5 K 2024-11-14T07:27:47,672 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/B/bfa6fc27a6b0415e82227431925f9ba2 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/bfa6fc27a6b0415e82227431925f9ba2 2024-11-14T07:27:47,674 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/bfa6fc27a6b0415e82227431925f9ba2, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T07:27:47,675 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/.tmp/C/19d52ad6512047d68ef11a042e466676 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/19d52ad6512047d68ef11a042e466676 2024-11-14T07:27:47,677 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/19d52ad6512047d68ef11a042e466676, entries=150, sequenceid=386, filesize=12.0 K 2024-11-14T07:27:47,678 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4687db304ff704d35e9eef31fb5612b9 in 1646ms, sequenceid=386, compaction requested=false 2024-11-14T07:27:47,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e] to archive 2024-11-14T07:27:47,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:47,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/3c3287f7aa604065b684862a6e29c057 2024-11-14T07:27:47,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/1a98a61907d541f3b468ff42cf20072a 2024-11-14T07:27:47,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/daa9345dfe9c46e193aca994387ba1dc 2024-11-14T07:27:47,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f80060f4450f43ba9fd886bc81b11210 2024-11-14T07:27:47,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9fb14bb87b75483eab44a956c8aec304 2024-11-14T07:27:47,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/0c7b05a1acd94082823142467e692113 2024-11-14T07:27:47,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/d7225aa61ee94b49bf18b10613e57678 2024-11-14T07:27:47,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f8daf1dba5ca4f6c8dc5813c7084ceef 2024-11-14T07:27:47,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/f13634efee6b4cdfbaf96c468ff17324 2024-11-14T07:27:47,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/95747da8bd3a4edc87e261ef05ce32d7 2024-11-14T07:27:47,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/27b1e4c8fa724086a60c2fc0f244c694 2024-11-14T07:27:47,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/94d29017ea21498c84c993404bde0b8e 2024-11-14T07:27:47,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/7f7a052cc6a84d4fb5596b524798473d 2024-11-14T07:27:47,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4b7eb498abe64cffa9dcb5325a825d8e 2024-11-14T07:27:47,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/c18819c84b2f44fcafbba629c3ecbb1d 2024-11-14T07:27:47,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/4ffe0af320e043e5aef1f02a4ef74069 2024-11-14T07:27:47,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/844280e953d246f792436d56efeddbca 2024-11-14T07:27:47,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/dd127c5ff677400ab70fef00de3c3728 2024-11-14T07:27:47,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/8ebfe4dccca8495f96f6356e3da2fe78 2024-11-14T07:27:47,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a3043eeef35a4ebbb5566fc47e5c51e9 2024-11-14T07:27:47,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/a36ea6f4fc554019aa48b7461a328bfb 2024-11-14T07:27:47,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/9b39404d546247b899976155daa76bd1 2024-11-14T07:27:47,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/695307d67770400eb3d3ac1586eea7ed 2024-11-14T07:27:47,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/abc2242399394141b4a8b0171d3d467d 2024-11-14T07:27:47,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/2a0f621966ae43338a1476ccab61df8f 2024-11-14T07:27:47,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/5acb4d760c54485daf9fb9626280c19e 2024-11-14T07:27:47,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/b066a02039744d12846fe623b8f0d2f2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/9bff1737522943f3a8e0b46b6bef600a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2a507ce02b884be6a0423994fce6cf07, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f661f3119e5e4b6c87f870320072251d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/cbd0f9871481473f8770f47e9727faea, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0c4acdd249b3484fa932f1c987a7512f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7589c906370748de8bc2f3f1a8b27afd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3] to archive 2024-11-14T07:27:47,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:47,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/89074f46cf174c33a4e38f8619d1798e 2024-11-14T07:27:47,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7ccd46f6cdc348e18d3f5002559238b2 2024-11-14T07:27:47,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/b066a02039744d12846fe623b8f0d2f2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/b066a02039744d12846fe623b8f0d2f2 2024-11-14T07:27:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/619bfa93465f47b2af68140dd28708ec 2024-11-14T07:27:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f043d444fc8d43d2b9cf722a3014766d 2024-11-14T07:27:47,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/8463d6de4cda49158588479cbb97a54d 2024-11-14T07:27:47,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/9bff1737522943f3a8e0b46b6bef600a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/9bff1737522943f3a8e0b46b6bef600a 2024-11-14T07:27:47,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dde3caf788b481fa567700bc78f7219 2024-11-14T07:27:47,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/5dabe642dc004484b08109dc2b644d39 2024-11-14T07:27:47,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2a507ce02b884be6a0423994fce6cf07 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2a507ce02b884be6a0423994fce6cf07 2024-11-14T07:27:47,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/87d6bff8a6eb461cbf4560d2d8194752 2024-11-14T07:27:47,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/41490dee88324be1a7f74c05a7da4518 2024-11-14T07:27:47,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f661f3119e5e4b6c87f870320072251d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f661f3119e5e4b6c87f870320072251d 2024-11-14T07:27:47,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/2dbebe565ad84d40b084b67c9c05aa7e 2024-11-14T07:27:47,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d66edb3db0f8427e958410ce81b73d50 2024-11-14T07:27:47,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/cbd0f9871481473f8770f47e9727faea to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/cbd0f9871481473f8770f47e9727faea 2024-11-14T07:27:47,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d023d60af39c486f8f0dbf8c2102d55d 2024-11-14T07:27:47,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/84a5c12a8555445783343221a44ddb30 2024-11-14T07:27:47,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/f347a16d62154d628054d94b28762f09 2024-11-14T07:27:47,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0c4acdd249b3484fa932f1c987a7512f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0c4acdd249b3484fa932f1c987a7512f 2024-11-14T07:27:47,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/0f77baadfe4145f3b197fcc5190121a9 2024-11-14T07:27:47,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/4e078c701e414f17b2ea213d9bd8e366 2024-11-14T07:27:47,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7589c906370748de8bc2f3f1a8b27afd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/7589c906370748de8bc2f3f1a8b27afd 2024-11-14T07:27:47,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/75efbdc07afa4d4daf1333006fecbc6e 2024-11-14T07:27:47,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/707f391266d4462ca76001f6b2396575 2024-11-14T07:27:47,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/fa1e420b6bfb460fa713d07b37d58af3 2024-11-14T07:27:47,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/aacdaacf3e454aef91df891039e7b491, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/c1fe15ef40484af3ae0ecb57d4a174d7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/366a1498709f4c178211e185eb19ddb2, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/4359a2edff6a42629c72b67dcf6944ba, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/765f15fc772643369e35a2dad7ed9f91, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/41a6419a5fb74eb384b8d78991d74532, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b8eb62995a3f43caa5a361311cb272f8, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414] to archive 2024-11-14T07:27:47,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T07:27:47,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/d9ff4de211004ffc85125bdae8e0ec65 2024-11-14T07:27:47,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/04d24aac07004ff4abd96dc1ee5b29c0 2024-11-14T07:27:47,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/aacdaacf3e454aef91df891039e7b491 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/aacdaacf3e454aef91df891039e7b491 2024-11-14T07:27:47,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/71bb99e640a5433dbcb396c77c935264 2024-11-14T07:27:47,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/6a6c02f5654f43e385084021d610b868 2024-11-14T07:27:47,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/75a75f30b6eb47adbb98fb93cf57313f 2024-11-14T07:27:47,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/c1fe15ef40484af3ae0ecb57d4a174d7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/c1fe15ef40484af3ae0ecb57d4a174d7 2024-11-14T07:27:47,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/84b06297336d4b1d8901104bf81af6d7 2024-11-14T07:27:47,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a27cd91c7ccb4958acbc3c6c4a1bdca1 2024-11-14T07:27:47,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/366a1498709f4c178211e185eb19ddb2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/366a1498709f4c178211e185eb19ddb2 2024-11-14T07:27:47,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/74349996f0e74e90aeada966c32a8c62 2024-11-14T07:27:47,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b5b081135ea740269b2b01578d65d88d 2024-11-14T07:27:47,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/4359a2edff6a42629c72b67dcf6944ba to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/4359a2edff6a42629c72b67dcf6944ba 2024-11-14T07:27:47,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b1dc73c929ac4c2e849e73ef2221b4dd 2024-11-14T07:27:47,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/95d376f6f4f44332bb5db66e82d8bd40 2024-11-14T07:27:47,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/765f15fc772643369e35a2dad7ed9f91 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/765f15fc772643369e35a2dad7ed9f91 2024-11-14T07:27:47,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/e34f5cc1bfcf4f4f9dc30c61ad5038d1 2024-11-14T07:27:47,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a812b7dd0fd649cbbe99911b5bbd8148 2024-11-14T07:27:47,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/0344a6a1952d4d79a6e309fe5e19bd9b 2024-11-14T07:27:47,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/41a6419a5fb74eb384b8d78991d74532 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/41a6419a5fb74eb384b8d78991d74532 2024-11-14T07:27:47,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/8fba9d79921a4eada990971d0190c2e9 2024-11-14T07:27:47,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b9fe78fe986c4373a4fda8496dcc750a 2024-11-14T07:27:47,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b8eb62995a3f43caa5a361311cb272f8 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/b8eb62995a3f43caa5a361311cb272f8 2024-11-14T07:27:47,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/a51a43b5977b4effb2bf990b7f44bb7c 2024-11-14T07:27:47,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f8818260abc49e38e3d2fe74370b1af 2024-11-14T07:27:47,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/00144180bd17468e9059c2d6551db414 2024-11-14T07:27:47,740 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits/389.seqid, newMaxSeqId=389, maxSeqId=4 2024-11-14T07:27:47,740 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. 2024-11-14T07:27:47,740 DEBUG [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for 4687db304ff704d35e9eef31fb5612b9: Waiting for close lock at 1731569266032Running coprocessor pre-close hooks at 1731569266032Disabling compacts and flushes for region at 1731569266032Disabling writes for close at 1731569266032Obtaining lock to block concurrent updates at 1731569266032Preparing flush snapshotting stores in 4687db304ff704d35e9eef31fb5612b9 at 1731569266032Finished memstore snapshotting TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731569266033 (+1 ms)Flushing stores of TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9. at 1731569266033Flushing 4687db304ff704d35e9eef31fb5612b9/A: creating writer at 1731569266033Flushing mob file 4687db304ff704d35e9eef31fb5612b9/A: appending metadata at 1731569266037 (+4 ms)Flushing mob file 4687db304ff704d35e9eef31fb5612b9/A: closing flushed file at 1731569266037Flushing 4687db304ff704d35e9eef31fb5612b9/A: appending metadata at 1731569266444 (+407 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/A: closing flushed file at 1731569266445 (+1 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/B: creating writer at 1731569266851 (+406 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/B: appending metadata at 1731569266853 (+2 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/B: closing flushed file at 1731569266853Flushing 4687db304ff704d35e9eef31fb5612b9/C: creating writer at 1731569267259 (+406 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/C: appending metadata at 1731569267261 (+2 ms)Flushing 4687db304ff704d35e9eef31fb5612b9/C: closing flushed file at 1731569267261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b6e36d3: reopening flushed file at 1731569267668 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eb8809b: reopening flushed file at 1731569267672 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb02a8c: reopening flushed file at 1731569267674 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4687db304ff704d35e9eef31fb5612b9 in 1646ms, sequenceid=386, compaction requested=false at 1731569267678 (+4 ms)Writing region close event to WAL at 1731569267737 (+59 ms)Running coprocessor post-close hooks at 1731569267740 (+3 ms)Closed at 1731569267740 2024-11-14T07:27:47,742 INFO [RS_CLOSE_REGION-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed 4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,742 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=4687db304ff704d35e9eef31fb5612b9, regionState=CLOSED 2024-11-14T07:27:47,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 because future has completed 2024-11-14T07:27:47,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-14T07:27:47,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure 4687db304ff704d35e9eef31fb5612b9, server=c7aa6e2790b3,46235,1731569085523 in 1.8650 sec 2024-11-14T07:27:47,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-11-14T07:27:47,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4687db304ff704d35e9eef31fb5612b9, UNASSIGN in 1.8700 sec 2024-11-14T07:27:47,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-14T07:27:47,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8740 sec 2024-11-14T07:27:47,749 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731569267749"}]},"ts":"1731569267749"} 2024-11-14T07:27:47,750 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T07:27:47,750 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T07:27:47,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9320 sec 2024-11-14T07:27:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T07:27:47,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:47,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T07:27:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:47,967 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-14T07:27:47,967 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:47,969 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,970 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C, FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits] 2024-11-14T07:27:47,976 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/82db3c6f24d74946b390a4d7b1d7b290 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/82db3c6f24d74946b390a4d7b1d7b290 2024-11-14T07:27:47,977 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/af3d5280de7f4004966d6bd3436bf670 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/A/af3d5280de7f4004966d6bd3436bf670 2024-11-14T07:27:47,979 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/bfa6fc27a6b0415e82227431925f9ba2 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/bfa6fc27a6b0415e82227431925f9ba2 2024-11-14T07:27:47,979 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d31e8a011a5e41baa762c7ea4c133351 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/B/d31e8a011a5e41baa762c7ea4c133351 2024-11-14T07:27:47,981 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/19d52ad6512047d68ef11a042e466676 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/19d52ad6512047d68ef11a042e466676 2024-11-14T07:27:47,981 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f1b67f7417d4f748cd5b3768479fe8b to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/C/5f1b67f7417d4f748cd5b3768479fe8b 2024-11-14T07:27:47,982 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits/389.seqid to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9/recovered.edits/389.seqid 2024-11-14T07:27:47,983 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/default/TestAcidGuarantees/4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,983 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T07:27:47,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:27:47,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T07:27:47,986 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412b3ea5a55024f2dabdc17742da8ca3f_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412b3ea5a55024f2dabdc17742da8ca3f_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412fd109e694245338207a9fa1733db10_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111412fd109e694245338207a9fa1733db10_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f3fb90fb3e54582b4392df28aa20062_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141f3fb90fb3e54582b4392df28aa20062_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141fad67693bc74aa49b4559aaadea8aea_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141fad67693bc74aa49b4559aaadea8aea_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142230ae5defc541328c30991b40ddcd39_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142230ae5defc541328c30991b40ddcd39_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c855211d2fe4a78b662eddbf21e91a1_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143c855211d2fe4a78b662eddbf21e91a1_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bf713e6f6824eae82bb779e737cf9e0_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144bf713e6f6824eae82bb779e737cf9e0_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145fc9f4a63ccc4368ade34aff55a9e2db_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145fc9f4a63ccc4368ade34aff55a9e2db_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114646d63e99f3e45d5be4177b9fbfb8cf1_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114646d63e99f3e45d5be4177b9fbfb8cf1_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,993 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146974f894ec234673a00d67aa24182dcd_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146974f894ec234673a00d67aa24182dcd_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,993 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f476a0c921249f3b039eea1b353b365_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146f476a0c921249f3b039eea1b353b365_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,994 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111479d7e843a7794f70ba88d37ddfe35db8_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111479d7e843a7794f70ba88d37ddfe35db8_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,995 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480239daf3f16493c8e73fee9bc28b612_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480239daf3f16493c8e73fee9bc28b612_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,995 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484155ebd707c4e848cd87073d2acb64c_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111484155ebd707c4e848cd87073d2acb64c_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,996 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111488d005ce2b2a46ffb9ceef42ea78b80b_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111488d005ce2b2a46ffb9ceef42ea78b80b_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a770e08957343d695dbc5e8893de847_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148a770e08957343d695dbc5e8893de847_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ab103e9225d444f9c692352f752b697_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ab103e9225d444f9c692352f752b697_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,998 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e94854655c4c4d0b85528b20f4f344ac_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e94854655c4c4d0b85528b20f4f344ac_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,998 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee4c67e41c4648d2bda7ca7673593997_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee4c67e41c4648d2bda7ca7673593997_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa5586ebcb264350a0bd3b9a094ee91c_4687db304ff704d35e9eef31fb5612b9 to hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa5586ebcb264350a0bd3b9a094ee91c_4687db304ff704d35e9eef31fb5612b9 2024-11-14T07:27:47,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T07:27:48,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:48,003 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T07:27:48,005 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T07:27:48,006 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:48,006 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T07:27:48,006 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731569268006"}]},"ts":"9223372036854775807"} 2024-11-14T07:27:48,008 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T07:27:48,008 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4687db304ff704d35e9eef31fb5612b9, NAME => 'TestAcidGuarantees,,1731569231595.4687db304ff704d35e9eef31fb5612b9.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T07:27:48,008 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T07:27:48,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731569268008"}]},"ts":"9223372036854775807"} 2024-11-14T07:27:48,010 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T07:27:48,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T07:27:48,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-14T07:27:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-14T07:27:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T07:27:48,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T07:27:48,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 238), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=5405 (was 5465) 2024-11-14T07:27:48,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T07:27:48,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:27:48,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:27:48,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:48,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:48,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T07:27:48,082 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T07:27:48,082 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-14T07:27:48,083 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c7aa6e2790b3,42927,1731569084750 2024-11-14T07:27:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T07:27:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T07:27:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:27:48,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:27:48,133 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T07:27:48,134 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T07:27:48,134 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:27:48,134 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:27:48,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:48,134 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T07:27:48,134 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c7aa6e2790b3,46235,1731569085523' ***** 2024-11-14T07:27:48,134 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T07:27:48,135 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(959): stopping server c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T07:27:48,135 INFO [RS:0;c7aa6e2790b3:46235 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c7aa6e2790b3:46235. 2024-11-14T07:27:48,135 DEBUG [RS:0;c7aa6e2790b3:46235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T07:27:48,135 DEBUG [RS:0;c7aa6e2790b3:46235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:48,136 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T07:27:48,136 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T07:27:48,136 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T07:27:48,136 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T07:27:48,136 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T07:27:48,136 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T07:27:48,137 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T07:27:48,137 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:48,137 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T07:27:48,137 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T07:27:48,137 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T07:27:48,137 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T07:27:48,137 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T07:27:48,145 INFO [regionserver/c7aa6e2790b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T07:27:48,158 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/info/e99045a8bf7b4331b5db1e0ccd033ba1 is 95, key is TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4./info:/1731569114864/DeleteFamily/seqid=0 2024-11-14T07:27:48,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742526_1702 (size=6514) 2024-11-14T07:27:48,337 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:48,537 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:48,561 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/info/e99045a8bf7b4331b5db1e0ccd033ba1 2024-11-14T07:27:48,583 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/ns/12cab4bea70a4bef9eb7a5ae576489c3 is 93, key is TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4./ns:/1731569114829/DeleteFamily/seqid=0 2024-11-14T07:27:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742527_1703 (size=5906) 2024-11-14T07:27:48,737 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:48,900 INFO [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T07:27:48,900 INFO [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T07:27:48,900 INFO [regionserver/c7aa6e2790b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-14T07:27:48,937 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:48,987 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/ns/12cab4bea70a4bef9eb7a5ae576489c3 2024-11-14T07:27:49,003 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/rep_barrier/358a74426a0940ed968464a7a7d7fa1d is 102, key is TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4./rep_barrier:/1731569114829/DeleteFamily/seqid=0 2024-11-14T07:27:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742528_1704 (size=6025) 2024-11-14T07:27:49,138 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:49,338 DEBUG [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T07:27:49,406 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/rep_barrier/358a74426a0940ed968464a7a7d7fa1d 2024-11-14T07:27:49,424 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/table/b6dcc7de624344399ff62065bed052a6 is 96, key is TestAcidGuarantees,,1731569089951.eb8aa6a5cb82db71eb9aa5187f031ed4./table:/1731569114829/DeleteFamily/seqid=0 2024-11-14T07:27:49,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742529_1705 (size=5936) 2024-11-14T07:27:49,427 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/table/b6dcc7de624344399ff62065bed052a6 2024-11-14T07:27:49,430 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/info/e99045a8bf7b4331b5db1e0ccd033ba1 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/info/e99045a8bf7b4331b5db1e0ccd033ba1 2024-11-14T07:27:49,433 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/info/e99045a8bf7b4331b5db1e0ccd033ba1, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T07:27:49,433 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/ns/12cab4bea70a4bef9eb7a5ae576489c3 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/ns/12cab4bea70a4bef9eb7a5ae576489c3 2024-11-14T07:27:49,436 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/ns/12cab4bea70a4bef9eb7a5ae576489c3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T07:27:49,437 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/rep_barrier/358a74426a0940ed968464a7a7d7fa1d as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/rep_barrier/358a74426a0940ed968464a7a7d7fa1d 2024-11-14T07:27:49,439 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/rep_barrier/358a74426a0940ed968464a7a7d7fa1d, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T07:27:49,439 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/.tmp/table/b6dcc7de624344399ff62065bed052a6 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/table/b6dcc7de624344399ff62065bed052a6 2024-11-14T07:27:49,446 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/table/b6dcc7de624344399ff62065bed052a6, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T07:27:49,447 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false 2024-11-14T07:27:49,450 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T07:27:49,451 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T07:27:49,451 INFO [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T07:27:49,451 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731569268136Running coprocessor pre-close hooks at 1731569268137 (+1 ms)Disabling compacts and flushes for region at 1731569268137Disabling writes for close at 1731569268137Obtaining lock to block concurrent updates at 1731569268137Preparing flush snapshotting stores in 1588230740 at 1731569268137Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731569268137Flushing stores of hbase:meta,,1.1588230740 at 1731569268138 (+1 ms)Flushing 1588230740/info: creating writer at 1731569268138Flushing 1588230740/info: appending metadata at 1731569268157 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731569268157Flushing 1588230740/ns: creating writer at 1731569268565 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731569268583 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731569268583Flushing 1588230740/rep_barrier: creating writer at 1731569268989 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731569269002 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731569269002Flushing 1588230740/table: creating writer at 1731569269409 (+407 ms)Flushing 1588230740/table: appending metadata at 1731569269423 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731569269423Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@143e52bb: reopening flushed file at 1731569269430 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b270feb: reopening flushed file at 1731569269433 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40b2f50c: reopening flushed file at 1731569269436 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@515cc7cf: reopening flushed file at 1731569269439 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1310ms, sequenceid=90, compaction requested=false at 1731569269447 (+8 ms)Writing region close event to WAL at 1731569269448 (+1 ms)Running coprocessor post-close hooks at 1731569269450 (+2 ms)Closed at 1731569269451 (+1 ms) 2024-11-14T07:27:49,451 DEBUG [RS_CLOSE_META-regionserver/c7aa6e2790b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T07:27:49,538 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(976): stopping server c7aa6e2790b3,46235,1731569085523; all regions closed. 2024-11-14T07:27:49,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741834_1010 (size=25344) 2024-11-14T07:27:49,545 DEBUG [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/oldWALs 2024-11-14T07:27:49,545 INFO [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c7aa6e2790b3%2C46235%2C1731569085523.meta:.meta(num 1731569089478) 2024-11-14T07:27:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741833_1009 (size=16041996) 2024-11-14T07:27:49,549 DEBUG [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/oldWALs 2024-11-14T07:27:49,549 INFO [RS:0;c7aa6e2790b3:46235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c7aa6e2790b3%2C46235%2C1731569085523:(num 1731569088998) 2024-11-14T07:27:49,549 DEBUG [RS:0;c7aa6e2790b3:46235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T07:27:49,549 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T07:27:49,549 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T07:27:49,550 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.ChoreService(370): Chore service for: regionserver/c7aa6e2790b3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T07:27:49,550 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T07:27:49,550 INFO [regionserver/c7aa6e2790b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T07:27:49,550 INFO [RS:0;c7aa6e2790b3:46235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46235 2024-11-14T07:27:49,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c7aa6e2790b3,46235,1731569085523 2024-11-14T07:27:49,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T07:27:49,598 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T07:27:49,609 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c7aa6e2790b3,46235,1731569085523] 2024-11-14T07:27:49,619 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c7aa6e2790b3,46235,1731569085523 already deleted, retry=false 2024-11-14T07:27:49,619 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c7aa6e2790b3,46235,1731569085523 expired; onlineServers=0 2024-11-14T07:27:49,619 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c7aa6e2790b3,42927,1731569084750' ***** 2024-11-14T07:27:49,619 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T07:27:49,619 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T07:27:49,619 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T07:27:49,620 DEBUG [M:0;c7aa6e2790b3:42927 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T07:27:49,620 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T07:27:49,620 DEBUG [M:0;c7aa6e2790b3:42927 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T07:27:49,620 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731569088636 {}] cleaner.HFileCleaner(306): Exit Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.small.0-1731569088636,5,FailOnTimeoutGroup] 2024-11-14T07:27:49,620 DEBUG [master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731569088635 {}] cleaner.HFileCleaner(306): Exit Thread[master/c7aa6e2790b3:0:becomeActiveMaster-HFileCleaner.large.0-1731569088635,5,FailOnTimeoutGroup] 2024-11-14T07:27:49,620 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.ChoreService(370): Chore service for: master/c7aa6e2790b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T07:27:49,620 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T07:27:49,620 DEBUG [M:0;c7aa6e2790b3:42927 {}] master.HMaster(1795): Stopping service threads 2024-11-14T07:27:49,620 INFO [M:0;c7aa6e2790b3:42927 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T07:27:49,620 INFO [M:0;c7aa6e2790b3:42927 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T07:27:49,621 ERROR [M:0;c7aa6e2790b3:42927 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T07:27:49,621 INFO [M:0;c7aa6e2790b3:42927 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T07:27:49,621 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T07:27:49,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T07:27:49,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T07:27:49,630 DEBUG [M:0;c7aa6e2790b3:42927 {}] zookeeper.ZKUtil(347): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T07:27:49,630 WARN [M:0;c7aa6e2790b3:42927 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T07:27:49,630 INFO [M:0;c7aa6e2790b3:42927 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/.lastflushedseqids 2024-11-14T07:27:49,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742530_1706 (size=72) 2024-11-14T07:27:49,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:27:49,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46235-0x101383b61db0001, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:27:49,709 INFO [RS:0;c7aa6e2790b3:46235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T07:27:49,710 INFO [RS:0;c7aa6e2790b3:46235 {}] regionserver.HRegionServer(1031): Exiting; stopping=c7aa6e2790b3,46235,1731569085523; zookeeper connection closed. 2024-11-14T07:27:49,710 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45d3520e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45d3520e 2024-11-14T07:27:49,710 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T07:27:50,039 INFO [M:0;c7aa6e2790b3:42927 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T07:27:50,040 INFO [M:0;c7aa6e2790b3:42927 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T07:27:50,040 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T07:27:50,040 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:27:50,040 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:27:50,040 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T07:27:50,040 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:27:50,040 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=793.22 KB heapSize=977.12 KB 2024-11-14T07:27:50,054 DEBUG [M:0;c7aa6e2790b3:42927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/688344acd2d54803ae3f81aafa9053fa is 82, key is hbase:meta,,1/info:regioninfo/1731569089566/Put/seqid=0 2024-11-14T07:27:50,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742531_1707 (size=5672) 2024-11-14T07:27:50,458 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2277 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/688344acd2d54803ae3f81aafa9053fa 2024-11-14T07:27:50,485 DEBUG [M:0;c7aa6e2790b3:42927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7e09388a99543d59e804e2c5ed7600b is 2276, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1731569118073/Put/seqid=0 2024-11-14T07:27:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742532_1708 (size=45611) 2024-11-14T07:27:50,889 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792.67 KB at sequenceid=2277 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7e09388a99543d59e804e2c5ed7600b 2024-11-14T07:27:50,893 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7e09388a99543d59e804e2c5ed7600b 2024-11-14T07:27:50,907 DEBUG [M:0;c7aa6e2790b3:42927 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29b18fe94ec74eadb43b42b18fb9b9e4 is 69, key is c7aa6e2790b3,46235,1731569085523/rs:state/1731569088679/Put/seqid=0 2024-11-14T07:27:50,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073742533_1709 (size=5156) 2024-11-14T07:27:51,311 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2277 (bloomFilter=true), to=hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29b18fe94ec74eadb43b42b18fb9b9e4 2024-11-14T07:27:51,314 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/688344acd2d54803ae3f81aafa9053fa as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/688344acd2d54803ae3f81aafa9053fa 2024-11-14T07:27:51,317 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/688344acd2d54803ae3f81aafa9053fa, entries=8, sequenceid=2277, filesize=5.5 K 2024-11-14T07:27:51,318 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7e09388a99543d59e804e2c5ed7600b as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7e09388a99543d59e804e2c5ed7600b 2024-11-14T07:27:51,321 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7e09388a99543d59e804e2c5ed7600b 2024-11-14T07:27:51,321 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7e09388a99543d59e804e2c5ed7600b, entries=178, sequenceid=2277, filesize=44.5 K 2024-11-14T07:27:51,322 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29b18fe94ec74eadb43b42b18fb9b9e4 as hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29b18fe94ec74eadb43b42b18fb9b9e4 2024-11-14T07:27:51,325 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37887/user/jenkins/test-data/d092f897-94b3-0245-b8eb-4ebfb0909b59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29b18fe94ec74eadb43b42b18fb9b9e4, entries=1, sequenceid=2277, filesize=5.0 K 2024-11-14T07:27:51,326 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(3140): Finished flush of dataSize ~793.22 KB/812261, heapSize ~976.82 KB/1000264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1286ms, sequenceid=2277, compaction requested=false 2024-11-14T07:27:51,327 INFO [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T07:27:51,327 DEBUG [M:0;c7aa6e2790b3:42927 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731569270040Disabling compacts and flushes for region at 1731569270040Disabling writes for close at 1731569270040Obtaining lock to block concurrent updates at 1731569270040Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731569270040Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=812261, getHeapSize=1000504, getOffHeapSize=0, getCellsCount=2480 at 1731569270040Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731569270041 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731569270041Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731569270054 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731569270054Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731569270461 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731569270484 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731569270484Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731569270893 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731569270907 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731569270907Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4314db: reopening flushed file at 1731569271313 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b032326: reopening flushed file at 1731569271317 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bd30773: reopening flushed file at 1731569271321 (+4 ms)Finished flush of dataSize ~793.22 KB/812261, heapSize ~976.82 KB/1000264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1286ms, sequenceid=2277, compaction requested=false at 1731569271326 (+5 ms)Writing region close event to WAL at 1731569271327 (+1 ms)Closed at 1731569271327 2024-11-14T07:27:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42715 is added to blk_1073741830_1006 (size=961497) 2024-11-14T07:27:51,330 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T07:27:51,330 INFO [M:0;c7aa6e2790b3:42927 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T07:27:51,330 INFO [M:0;c7aa6e2790b3:42927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42927 2024-11-14T07:27:51,330 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T07:27:51,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:27:51,477 INFO [M:0;c7aa6e2790b3:42927 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T07:27:51,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42927-0x101383b61db0000, quorum=127.0.0.1:60845, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T07:27:51,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T07:27:51,483 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T07:27:51,483 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T07:27:51,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T07:27:51,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.log.dir/,STOPPED} 2024-11-14T07:27:51,486 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T07:27:51,486 WARN [BP-1617860635-172.17.0.2-1731569080462 heartbeating to localhost/127.0.0.1:37887 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T07:27:51,486 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T07:27:51,486 WARN [BP-1617860635-172.17.0.2-1731569080462 heartbeating to localhost/127.0.0.1:37887 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1617860635-172.17.0.2-1731569080462 (Datanode Uuid 799d9034-1b08-4be6-9f35-4a763b88be90) service to localhost/127.0.0.1:37887 2024-11-14T07:27:51,489 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data1/current/BP-1617860635-172.17.0.2-1731569080462 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T07:27:51,489 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/cluster_99b68c0e-d550-3bbb-f922-2b5eb739da8d/data/data2/current/BP-1617860635-172.17.0.2-1731569080462 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T07:27:51,490 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T07:27:51,495 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T07:27:51,496 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T07:27:51,496 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T07:27:51,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T07:27:51,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/e1c3dc6a-e116-bc6a-acac-7cf37419cb52/hadoop.log.dir/,STOPPED} 2024-11-14T07:27:51,515 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T07:27:51,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down